diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
index 2ed66ad7..4e10236c 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
@@ -1,7 +1,7 @@
{
"schema_version": 2,
"note": "Developer adjudications of judge verdicts. An entry either resolves a non-final judge class to a final one (llm_error, parse_contract_failure) or affirms prompt_ambiguity, which is allowed only together with excluded_from_scoring: true, meaning the output is removed from scoring for every model and listed in the run's reference_exclusions.json (the freezer verifies the two records agree). Each entry keeps the judge's verdict verbatim beside the decision and the reasoning. Applied to the row annotations and case notes by scripts/apply_adjudications.py before export.",
- "date_conventions": "adjudicated_on names the audit wave that made the decision (2026-09-05, 2026-09-22, 2026-09-29 or 2026-10-05). It carries no time of day or time zone. The 2026-09-05, 2026-09-22 and 2026-09-29 waves' decisions were written up to the day each wave's release was committed (2026-09-05, 2026-09-23 and 2026-09-30); the 2026-09-29 wave's reference sweep began on 2026-09-29 UTC, before its decisions were written. The 2026-10-05 wave's decisions follow Max's rulings of 2026-10-05 US Eastern time (2026-10-06 UTC) and were written on 2026-10-06 UTC. judged_on_utc, judge_rejudged_on and each judge_previous item's judged_on are UTC days, read from the verdict's sidecar (verdict.meta.json judged_at_utc). The top-level judge fields name the case's verdict in this release's audit tree, dated by judge_rejudged_on when a later wave judged the case again and by judged_on_utc otherwise, and judge_previous keeps the verdict that re-judge replaced. A wave's own judge runs could finish after one of its decisions was first written, so a recorded verdict can postdate adjudicated_on up to the last day its wave's decisions were written. A flag an earlier judge run raised stays recorded and names that run in judge_reference_suspect_source, at the top level or on the judge_previous item it belongs to. Where a later wave replaced the verdict a decision reviewed without keeping it, adjudicated_verdict gives that verdict as the decision's own release recorded it.",
+ "date_conventions": "adjudicated_on names the audit wave that made the decision (2026-09-05, 2026-09-22, 2026-09-29, 2026-10-05, 2026-10-06 or 2026-10-10). It carries no time of day or time zone. The 2026-09-05, 2026-09-22 and 2026-09-29 waves' decisions were written up to the day each wave's release was committed (2026-09-05, 2026-09-23 and 2026-09-30); the 2026-09-29 wave's reference sweep began on 2026-09-29 UTC, before its decisions were written. The 2026-10-05 wave's decisions follow Max's rulings of 2026-10-05 US Eastern time (2026-10-06 UTC) and were written on 2026-10-06 UTC. The 2026-10-06 wave's decisions follow Max's rulings of 2026-10-06 (17:11 UTC) and were written on 2026-10-09 UTC. The 2026-10-10 wave's decisions exclude the outputs the references' engine upgrade moved onto an input the prompt does not state, and were written on 2026-10-10 UTC. judged_on_utc, judge_rejudged_on and each judge_previous item's judged_on are UTC days, read from the verdict's sidecar (verdict.meta.json judged_at_utc). The top-level judge fields name the case's verdict in this release's audit tree, dated by judge_rejudged_on when a later wave judged the case again and by judged_on_utc otherwise, and judge_previous keeps the verdict that re-judge replaced. A wave's own judge runs could finish after one of its decisions was first written, so a recorded verdict can postdate adjudicated_on up to the last day its wave's decisions were written. A flag an earlier judge run raised stays recorded and names that run in judge_reference_suspect_source, at the top level or on the judge_previous item it belongs to. Where a later wave replaced the verdict a decision reviewed without keeping it, adjudicated_verdict gives that verdict as the decision's own release recorded it.",
"adjudications": [
{
"country": "us",
@@ -54,7 +54,7 @@
"scenario_id": "scenario_074",
"variable": "head_medicare_eligible",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "age_disability",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -65,7 +65,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5",
@@ -80,6 +80,13 @@
"judge_failure_subtype": "age_disability",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "age_disability",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -129,7 +136,7 @@
"scenario_id": "scenario_084",
"variable": "head_medicare_eligible",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "age_disability",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -140,7 +147,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -155,6 +162,13 @@
"judge_failure_subtype": "age_disability",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "age_disability",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
],
"adjudicated_verdict": {
@@ -170,7 +184,7 @@
"scenario_id": "scenario_023",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-08",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "age_disability",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -181,7 +195,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04).",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -196,6 +210,13 @@
"judge_failure_subtype": "age_disability",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "age_disability",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
],
"adjudicated_verdict": {
@@ -211,7 +232,7 @@
"scenario_id": "scenario_057",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -225,7 +246,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -240,6 +261,14 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
],
"adjudicated_verdict": {
@@ -255,7 +284,7 @@
"scenario_id": "scenario_100",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -266,7 +295,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification.",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -281,6 +310,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
],
"adjudicated_verdict": {
@@ -296,7 +332,7 @@
"scenario_id": "scenario_057",
"variable": "ssi",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "categorical_eligibility",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -310,7 +346,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -325,6 +361,14 @@
"judge_failure_subtype": "categorical_eligibility",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
],
"adjudicated_verdict": {
@@ -437,40 +481,6 @@
"reasoning": "The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.",
"judged_on_utc": "2026-09-05"
},
- {
- "country": "us",
- "scenario_id": "scenario_003",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "reasoning": "The frozen reference is 22,154.70; the corrected value is 22,400.65. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29"
- }
- ]
- },
{
"country": "us",
"scenario_id": "scenario_005",
@@ -487,7 +497,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
"reasoning": "The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -502,41 +512,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_005",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076",
- "reasoning": "The frozen reference is 41,051.51; the corrected value is 40,920.40. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
},
{
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29"
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -557,7 +539,7 @@
"reference_basis": "Idaho Code 63-3022P; Idaho Form 39R",
"reasoning": "The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -572,41 +554,14 @@
"judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_008",
- "variable": "payroll_tax",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "payroll_tax_base",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates",
- "reasoning": "The frozen reference is 2,162.76; the corrected value is 2,276.66. PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "state_local_rule",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": true,
- "judged_on": "2026-09-29"
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -627,7 +582,7 @@
"reference_basis": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say",
"reasoning": "The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.",
"judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -643,62 +598,34 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_022",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "household_unit_or_filing_status",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "state_local_rule",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "Cal. R&TC 17024.5 (conformity date), 17076",
- "reasoning": "The frozen reference is 2,439.65; the corrected value is 1,968.81. PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
- "judged_on": "2026-09-29",
+ "judged_on": "2026-09-30",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
{
"country": "us",
- "scenario_id": "scenario_039",
- "variable": "federal_income_tax_before_refundable_credits",
+ "scenario_id": "scenario_042",
+ "variable": "state_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "adjudicated_failure_subtype": "state_local_rule",
"adjudicated_on": "2026-09-22",
"adjudicator": "developer",
"excluded_from_scoring": true,
"judge_reference_suspect": true,
"reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "reasoning": "The frozen reference is 1,345.51; the corrected value is 8,596.03. PolicyEngine leaves estate income out of gross income while counting it as qualified business income.",
- "judge_rejudged_on": "2026-09-30",
+ "reference_basis": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5",
+ "reasoning": "The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -713,16 +640,23 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_engine_defect",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30"
}
]
},
{
"country": "us",
- "scenario_id": "scenario_039",
- "variable": "state_income_tax_before_refundable_credits",
+ "scenario_id": "scenario_052",
+ "variable": "federal_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
"adjudicated_failure_source": "reference_engine_defect",
"adjudicated_failure_subtype": "taxable_income_or_deductions",
"adjudicated_on": "2026-09-22",
@@ -730,9 +664,9 @@
"excluded_from_scoring": true,
"judge_reference_suspect": true,
"reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "reasoning": "The frozen reference is 514.50; the corrected value is 1,975.80. PolicyEngine leaves estate income out of gross income while counting it as qualified business income.",
- "judge_rejudged_on": "2026-09-30",
+ "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
+ "reasoning": "The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -741,22 +675,29 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-23"
},
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-29"
+ },
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
- "judged_on": "2026-09-29"
+ "judged_on": "2026-09-30"
}
]
},
{
"country": "us",
- "scenario_id": "scenario_042",
+ "scenario_id": "scenario_053",
"variable": "state_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
"adjudicated_failure_source": "reference_engine_defect",
"adjudicated_failure_subtype": "state_local_rule",
"adjudicated_on": "2026-09-22",
@@ -764,49 +705,58 @@
"excluded_from_scoring": true,
"judge_reference_suspect": true,
"reference_verdict": "engine_defect",
- "reference_basis": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5",
- "reasoning": "The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.",
- "judge_rejudged_on": "2026-09-30",
+ "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R",
+ "reasoning": "The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": true,
"judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
{
"country": "us",
- "scenario_id": "scenario_049",
- "variable": "federal_income_tax_before_refundable_credits",
+ "scenario_id": "scenario_056",
+ "variable": "snap",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "state_local_rule",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "categorical_eligibility",
"adjudicated_on": "2026-09-22",
"adjudicator": "developer",
"excluded_from_scoring": true,
"judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "reasoning": "The frozen reference is 30,543.91; the corrected value is 30,702.59. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies",
+ "reasoning": "The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": true,
"judged_on": "2026-09-23"
},
@@ -816,110 +766,14 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_052",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "reasoning": "The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_053",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "state_local_rule",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R",
- "reasoning": "The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_056",
- "variable": "snap",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "prompt_ambiguity",
- "adjudicated_failure_subtype": "categorical_eligibility",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "unlisted_input",
- "reference_basis": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies",
- "reasoning": "The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
- "judged_on": "2026-09-29"
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -973,7 +827,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "the relationship of an adult tax dependent to the filers who claim them",
"reasoning": "The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).",
- "judge_rejudged_on": "2026-09-29",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5",
@@ -981,6 +835,13 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": false,
"judged_on": "2026-09-05"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
}
]
},
@@ -1018,76 +879,6 @@
}
]
},
- {
- "country": "us",
- "scenario_id": "scenario_064",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "reasoning": "The frozen reference is 4,439.29; the corrected value is 4,441.46. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_064",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": false,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "reasoning": "The frozen reference is 4,606.00; the corrected value is 4,599.62. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-23"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-29"
- }
- ]
- },
{
"country": "us",
"scenario_id": "scenario_067",
@@ -1132,7 +923,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04)",
"reasoning": "The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1147,6 +938,13 @@
"judge_failure_subtype": "categorical_eligibility",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -1167,7 +965,7 @@
"reference_basis": "M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21",
"reasoning": "The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes.",
"judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1183,40 +981,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_082",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": false,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "reasoning": "The frozen reference is 9,563.05; the corrected value is 9,564.92. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-29"
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1237,7 +1009,7 @@
"reference_basis": "Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5",
"reasoning": "The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1252,40 +1024,14 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_099",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "credit_phaseout",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": false,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "reasoning": "The frozen reference is 10,679.75; the corrected value is 10,711.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "credit_phaseout",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "credit_phaseout",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-29"
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1305,8 +1051,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076",
"reasoning": "The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-10",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1321,6 +1066,14 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1329,7 +1082,7 @@
"scenario_id": "scenario_100",
"variable": "federal_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "reference_engine_defect",
"adjudicated_failure_subtype": "credit_phaseout",
@@ -1340,9 +1093,8 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)",
"reasoning": "The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judged_on_utc": "2026-09-30",
- "judge_rejudged_on": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1350,6 +1102,14 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-22"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1370,7 +1130,7 @@
"reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)",
"reasoning": "The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1385,6 +1145,14 @@
"judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1412,7 +1180,7 @@
"variable": "state_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
+ "judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
"adjudicated_on": "2026-09-22",
@@ -1423,7 +1191,7 @@
"reference_basis": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits",
"reasoning": "The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1438,74 +1206,14 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_110",
- "variable": "federal_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": true,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "reasoning": "The frozen reference is 23,897.44; the corrected value is 25,747.77. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-23"
},
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
- "judge_reference_suspect": true,
- "judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_110",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": false,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "reasoning": "The frozen reference is 4,057.47; the corrected value is 4,236.81. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-23"
- },
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "thresholds_rates",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-29"
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1514,8 +1222,8 @@
"scenario_id": "scenario_112",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "categorical_eligibility",
"adjudicated_on": "2026-09-22",
@@ -1525,7 +1233,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "weekly_hours_worked_before_lsr",
"reasoning": "The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1541,6 +1250,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30"
}
]
},
@@ -1560,7 +1276,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the listed home mortgage interest is on the home the SNAP household occupies",
"reasoning": "The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1575,6 +1291,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -1594,8 +1317,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)",
"reasoning": "The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.",
- "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1611,6 +1333,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "credit_phaseout",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1619,8 +1349,8 @@
"scenario_id": "scenario_119",
"variable": "state_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "credit_phaseout",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "reference_engine_defect",
"adjudicated_failure_subtype": "credit_phaseout",
"adjudicated_on": "2026-09-22",
@@ -1630,7 +1360,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)",
"reasoning": "The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1646,6 +1376,13 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_subtype": "credit_phaseout",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30"
}
]
},
@@ -1665,7 +1402,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
"reasoning": "The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1680,40 +1417,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
- }
- ]
- },
- {
- "country": "us",
- "scenario_id": "scenario_120",
- "variable": "state_income_tax_before_refundable_credits",
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "adjudicated_failure_source": "reference_engine_defect",
- "adjudicated_failure_subtype": "taxable_income_or_deductions",
- "adjudicated_on": "2026-09-22",
- "adjudicator": "developer",
- "excluded_from_scoring": true,
- "judge_reference_suspect": false,
- "reference_verdict": "engine_defect",
- "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "reasoning": "The frozen reference is 11,917.18; the corrected value is 11,919.06. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-09-30",
- "judge_previous": [
- {
- "judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
- "judge_reference_suspect": false,
- "judged_on": "2026-09-23"
},
{
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "state_local_rule",
+ "judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
- "judged_on": "2026-09-29"
+ "judged_on": "2026-09-30"
}
]
},
@@ -1786,7 +1496,7 @@
"scenario_id": "scenario_056",
"variable": "state_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_source": "reference_engine_defect",
"judge_failure_subtype": "credit_phaseout",
"adjudicated_failure_source": "llm_error",
"adjudicated_failure_subtype": "credit_phaseout",
@@ -1796,7 +1506,7 @@
"reference_verdict": "affirmed",
"reference_basis": "N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32",
"reasoning": "New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails.",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1811,6 +1521,13 @@
"judge_failure_subtype": "credit_phaseout",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_subtype": "credit_phaseout",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30"
}
]
},
@@ -1829,8 +1546,7 @@
"reference_verdict": "affirmed",
"reference_basis": "26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12",
"reasoning": "The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law.",
- "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-09-29",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1838,6 +1554,14 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-09-23"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-29",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1857,7 +1581,7 @@
"reference_basis": "Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601",
"reasoning": "Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount.",
"judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1873,6 +1597,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1891,8 +1623,7 @@
"reference_verdict": "affirmed",
"reference_basis": "N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025",
"reasoning": "From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule.",
- "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1907,6 +1638,14 @@
"judge_failure_subtype": "categorical_eligibility",
"judge_reference_suspect": true,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1926,7 +1665,7 @@
"reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)",
"reasoning": "The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1943,6 +1682,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1952,7 +1699,7 @@
"variable": "federal_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "household_unit_or_filing_status",
"adjudicated_failure_source": "llm_error",
"adjudicated_failure_subtype": "taxable_income_or_deductions",
"adjudicated_on": "2026-09-22",
@@ -1962,7 +1709,7 @@
"reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7",
"reasoning": "The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1979,6 +1726,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1989,16 +1744,17 @@
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "thresholds_rates",
- "adjudicated_failure_source": "llm_error",
- "adjudicated_failure_subtype": "thresholds_rates",
- "adjudicated_on": "2026-09-22",
+ "adjudicated_failure_source": "reference_later_law",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "adjudicated_on": "2026-10-06",
"adjudicator": "developer",
+ "excluded_from_scoring": true,
"judge_reference_suspect": true,
- "reference_verdict": "regenerated",
- "reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7",
- "reasoning": "The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.",
+ "reference_verdict": "later_law",
+ "reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "reasoning": "The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2015,6 +1771,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -2034,7 +1798,18 @@
"reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14)",
"reasoning": "The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judged_on_utc": "2026-09-23"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-23",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ }
+ ]
},
{
"country": "us",
@@ -2070,7 +1845,7 @@
"reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A)",
"reasoning": "The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-09-30",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2087,6 +1862,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-09-30",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -2106,7 +1889,17 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
@@ -2124,8 +1917,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-09-30",
- "judge_rejudged_on": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2133,6 +1926,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -2152,8 +1952,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-09-30",
- "judge_rejudged_on": "2026-09-30",
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2161,6 +1961,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -2180,7 +1987,17 @@
"reference_verdict": "unlisted_input",
"reference_basis": "meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require",
"reasoning": "The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release.",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
@@ -2195,14 +2012,24 @@
"adjudicator": "developer",
"judge_reference_suspect": false,
"reasoning": "The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
"scenario_id": "scenario_008",
"variable": "state_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "reference_later_law",
"judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "llm_error",
"adjudicated_failure_subtype": "state_local_rule",
@@ -2210,8 +2037,8 @@
"adjudicator": "developer",
"judge_reference_suspect": false,
"reasoning": "The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.",
- "judged_on_utc": "2026-09-30",
- "judge_rejudged_on": "2026-09-30",
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2219,6 +2046,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-09-29"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
}
]
},
@@ -2228,7 +2062,7 @@
"variable": "federal_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "taxable_income_or_deductions",
"adjudicated_on": "2026-10-05",
@@ -2238,14 +2072,24 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set",
"reasoning": "The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).",
- "judged_on_utc": "2026-09-30"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
+ }
+ ]
},
{
"country": "us",
"scenario_id": "scenario_032",
"variable": "payroll_tax",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "prompt_ambiguity",
"judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
@@ -2256,14 +2100,24 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
"scenario_id": "scenario_043",
"variable": "payroll_tax",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "prompt_ambiguity",
"judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
@@ -2274,7 +2128,17 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-09-22"
+ "judged_on_utc": "2026-10-08",
+ "judge_rejudged_on": "2026-10-08",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-22"
+ }
+ ]
},
{
"country": "us",
@@ -2292,14 +2156,24 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set",
"reasoning": "The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).",
- "judged_on_utc": "2026-09-30"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
+ }
+ ]
},
{
"country": "us",
"scenario_id": "scenario_081",
"variable": "payroll_tax",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "prompt_ambiguity",
"judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
@@ -2310,14 +2184,24 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
"scenario_id": "scenario_082",
"variable": "payroll_tax",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "llm_error",
+ "judge_failure_source": "prompt_ambiguity",
"judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
@@ -2328,7 +2212,17 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-09-29"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-29"
+ }
+ ]
},
{
"country": "us",
@@ -2346,7 +2240,17 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either",
"reasoning": "The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b).",
- "judged_on_utc": "2026-09-30"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
+ }
+ ]
},
{
"country": "us",
@@ -2364,7 +2268,143 @@
"reference_verdict": "unlisted_input",
"reference_basis": "Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either",
"reasoning": "The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py).",
- "judged_on_utc": "2026-09-30"
+ "judged_on_utc": "2026-10-09",
+ "judge_rejudged_on": "2026-10-09",
+ "judge_previous": [
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-09-30"
+ }
+ ]
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "adjudicated_failure_source": "reference_later_law",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "adjudicated_on": "2026-10-06",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "later_law",
+ "reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "reasoning": "The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).",
+ "judged_on_utc": "2026-10-09"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_093",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_on": "2026-10-06",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "reasoning": "The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
+ "judged_on_utc": "2026-10-09"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_093",
+ "variable": "state_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_on": "2026-10-06",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "reasoning": "The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
+ "judged_on_utc": "2026-10-09"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_123",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_on": "2026-10-06",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "reasoning": "The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
+ "judged_on_utc": "2026-10-09"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_123",
+ "variable": "state_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_on": "2026-10-06",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "reasoning": "The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
+ "judged_on_utc": "2026-10-09"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "adjudicated_on": "2026-10-10",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "county of residence (Indiana county income tax)",
+ "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.",
+ "judged_on_utc": "2026-10-10"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "adjudicated_on": "2026-10-10",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "county of residence (Indiana county income tax)",
+ "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.",
+ "judged_on_utc": "2026-10-10"
}
]
}
diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
index 6bc1b7b9..44804810 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
@@ -1,46 +1,47 @@
country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Treated the head as a Qualifying Surviving Spouse even though there is no dependent child. It then applied the joint $32,000/$44,000 Social Security thresholds, which gave taxable SS of $11,542, plus the joint $32,300 standard deduction. Filing Single gives taxable SS of $18,542, total deductions of $24,150 and taxable income of $26,287."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the joint/QSS $32,200 standard deduction, $1,650 aged add-on and joint brackets. QSS requires a dependent child, so the head files Single with a $16,100 + $2,050 + $6,000 deduction and pays 12% on income above $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Counted the $48,000 of nontaxable VA benefits in gross income, took the full 85% of Social Security, and subtracted the ESI premiums even though there are no wages. It then set tax to $0 without support. The correct AGI is $50,437, and after the $24,150 of single/aged/senior deductions, $26,287 of taxable income remains."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Used the full 0.5 × SS ($13,048) as the first tier of the taxable Social Security formula instead of the lesser-of amount capped at $4,500. That produced the $22,182 cap instead of $18,542. It also ended the 10% bracket at $12,150 rather than the 2026 figure of $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Filed as QSS/MFJ without a qualifying dependent, taxed Social Security at the 85% cap, and then declared taxable income near zero even though its own figures left about $14,000 taxable. Single status with $24,150 of deductions leaves $26,287 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Chose Single status and the $24,150 of deductions correctly, but taxed Social Security at the 85% cap ($22,182) instead of $4,500 + 0.85 × $16,520 = $18,542. It then cut its own ~$3,300 result to $1,176 with no computation behind it."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Used the joint/QSS $32,200 standard deduction, $1,650 aged add-on and joint 10% bracket for a surviving spouse with no dependent child. That surviving spouse must file Single, with $24,150 of total deductions and the 12% rate above $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Assumed QSS status, which requires a dependent child, and so used MFJ Social Security thresholds, the 85% cap and a joint standard deduction of about $31,550. It also left out the $6,000 OBBBA senior deduction. Single filing gives AGI of $50,437 less $24,150 of deductions."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Filed as QSS with MFJ thresholds and a $33,850 joint deduction, and left out the $6,000 senior deduction. It then raised its own $2,023 result to $3,204 with a made-up 'higher bracket' adjustment. The correct Single computation gives $26,287 taxable and $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Computed Social Security with the single thresholds but took the joint/QSS $32,200 + $1,650 standard deduction and joint brackets. With no dependent child the head files Single: $16,100 + $2,050 + $6,000."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"AGI was right at $50,436, but it used the pre-OBBBA $15,400 single standard deduction and left out the $6,000 senior deduction. The correct 2026 total is $16,100 + $2,050 + $6,000 = $24,150. It also ended the 10% bracket at $12,025 instead of $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Taxed Social Security at the full 85% cap ($22,182) instead of the $18,542 the single-threshold formula gives. It also used a $17,350 standard deduction with no $6,000 senior deduction, which overstated taxable income by about $10,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA sunset and applied a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has no exemption, uses 10%/12% brackets, and adds the $18,150 aged standard deduction plus the $6,000 senior deduction. It also wrongly counted the ESI premiums as a deductible medical expense."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"AGI and the $18,150 single aged standard deduction were right, but it left out the $6,000 OBBBA senior deduction for filers 65 and older. It also ended the 10% bracket at $12,200 instead of $12,400, so it taxed $32,286 instead of $26,287."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA expired in 2026 and applied a personal exemption, 15% brackets and a medical deduction built from ESI premiums. OBBBA made the TCJA rates and standard deduction permanent and added the $6,000 senior deduction, which gives $24,150 of deductions taxed at 10%/12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of $37,990 leaves out the $6,000 senior deduction. Its $1,109 tax also does not follow from applying 2026 single brackets to that figure. The correct derivation is $50,437 AGI less $24,150 of deductions, giving $26,287 taxable and $2,906.45 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied reverted pre-TCJA law: itemized real estate taxes, a roughly $5,400 personal exemption and a 15% bracket. Under OBBBA-permanent law there is no exemption, the single aged standard deduction of $18,150 plus the $6,000 senior deduction beats itemizing, and the second bracket is 12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Assumed 2026 reverted to pre-TCJA rules, with a $5,050 personal exemption and a 15% bracket above $11,000. OBBBA made the TCJA structure permanent, so the $24,150 of standard, aged and senior deductions applies with 10%/12% brackets."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed income was below the deductions, but AGI including $18,542 of taxable Social Security is $50,437, against total single aged and senior deductions of only $24,150. That leaves $26,287 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Itemized real estate taxes and subtracted a personal exemption as if the TCJA had sunset. Under 2026 OBBBA law there is no personal exemption, the $18,150 standard deduction plus the $6,000 senior deduction applies, and rates are 10%/12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used pre-TCJA 2026 rules: itemized property tax, a personal exemption and 10%/15% brackets. OBBBA-permanent law instead gives $24,150 of standard, aged and senior deductions with 10%/12% brackets."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"AGI was right at $50,436, but it used a $17,000 single aged standard deduction and left out the $6,000 OBBBA senior deduction. The correct total is $16,100 + $2,050 + $6,000 = $24,150."
-us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Filing status and AGI were right, but it used an inflation-projected $17,510 pre-OBBBA standard deduction and left out the $6,000 senior deduction. That taxed $32,926 instead of $26,287, and its bracket boundary of $12,283 was not the 2026 figure of $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Counted the ESI premiums as a deductible medical expense and left out the $6,000 senior deduction, which applies whether or not the filer itemizes. Actual itemizable deductions come to $17,587.20, less than the $18,150 standard deduction, so the correct total is $24,150."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"Relied on a 'surviving-spouse rate structure' that does not apply without a dependent child, and claimed the deductions eliminate the tax. Single status gives $50,437 AGI less $24,150 of deductions, leaving $26,287 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Claimed the age-77 standard deduction offsets all income, but it ignored the $18,542 of taxable Social Security on top of $31,894 of other income. AGI of $50,437 minus $24,150 of deductions leaves $26,287 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse/joint Social Security thresholds, which gave $11,542 taxable SS, and the joint $32,200 standard deduction. Without a dependent child the head files Single: taxable SS is $18,542 and deductions total $24,150."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Framework and brackets were right, but it overstated taxable Social Security at $18,592 instead of $18,542.70, which put AGI at $50,486 and taxable income at $26,336 instead of $26,287.11. The $49 excess taxed at 12% adds about $5.89."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse (joint) thresholds, standard deduction and 10% bracket, which is consistent with $11,542 taxable SS and $3,586 taxable. QSS requires a dependent child, so the head files Single with $26,287 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the $33,850 joint/QSS age-adjusted standard deduction. The head has no dependent child and files Single, with a $18,150 standard deduction plus the $6,000 senior deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Overstated taxable income at $26,336 instead of $26,287.11 because it carried about $49 too much taxable Social Security. It then rounded, landing at $2,912 instead of $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Applied the surviving-spouse joint Social Security thresholds, which gave $11,542 taxable SS, and the $33,850 joint deduction. With no dependent child the head files Single, so taxable SS is $18,542 and taxable income is $26,287."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Claimed the senior deductions exceed taxable income, but AGI of $50,437 minus the $24,150 total of the standard, aged and senior deductions leaves $26,287 taxable at 10%/12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset law, with a $5,300 personal exemption and a 15% bracket, and also treated the head as QSS with a joint $23,200 10% bracket. Correct 2026 law files Single with $24,150 of standard, aged and senior deductions, 10% up to $12,400 and 12% above."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Assumed the TCJA expired, so it itemized property tax, took a personal exemption and used a 15% bracket. OBBBA made the TCJA structure permanent: the $18,150 standard deduction plus the $6,000 senior deduction applies, with 10%/12% brackets."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Taxable income of $26,286 was essentially correct, but it ended the 10% bracket at $12,200 instead of the 2026 single threshold of $12,400. That produced $2,910.32 instead of $1,240 + 12% × $13,887.11 = $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Assumed the TCJA sunset: it treated the SALT cap as gone, taxed at 15% above the first bracket, and took a medical deduction from the ESI premiums. Under 2026 OBBBA law the $18,150 standard deduction plus the $6,000 senior deduction applies, with 10%/12% brackets."
-us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No value or reasoning was returned for federal_income_tax_before_refundable_credits.
-us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Subtracted a $750 Credit for the Elderly and a Lifetime Learning credit to reach zero. The education credit has no qualifying expenses. The elderly credit's $5,000 initial amount is fully eliminated by nontaxable Social Security plus half of AGI above $7,500, so no nonrefundable credits apply against the $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Computed provisional income as $37,472 without adding half of Social Security, and dropped the $4,500 lower-tier amount, which gave only $2,951 of taxable SS. With half of SS included, provisional income is $50,520 and taxable SS is $4,500 + 0.85 × $16,520 = $18,542."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated Social Security as entirely nontaxable, even though provisional income of $50,520 far exceeds the base amount. It also used a joint/QSS $31,150 standard deduction without a dependent child and left out the $6,000 senior deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the single aged standard deduction eliminates taxable income, but $50,437 of AGI, including $18,542 of taxable Social Security, minus $24,150 of deductions leaves $26,287 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse even though there is no dependent child. That gave it the $32,000/$44,000 joint Social Security thresholds (taxable SS $11,542) and a joint $32,300 standard deduction. Single status yields $18,542.70 of taxable SS, an $18,150 standard deduction plus the $6,000 senior deduction, and $26,287.11 taxable, with part of it in the 12% bracket."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It computed taxable SS correctly at $18,542 but applied the $32,200 joint standard deduction for qualifying surviving spouse status, which requires a dependent child. The single deduction is $16,100 plus $2,050 aged plus $6,000 senior. So taxable income is $26,287, not $10,586, and part of it falls in the 12% bracket."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included the nontaxable $48,000 of veterans benefits in gross income and subtracted ESI premiums from AGI even though there are no wages. It then discarded its own roughly $6,600 tax estimate and asserted $0 with no supporting computation. The correct taxable income after the single standard, aged, and senior deductions is $26,287, which produces $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income falls below zero after deductions. In fact, AGI of $50,437 (including $18,542.70 of taxable Social Security) less the $18,150 single age-adjusted standard deduction and the $6,000 senior deduction leaves $26,287 taxable, which owes $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"In the taxable Social Security formula it used the full half of benefits ($13,048) instead of the lesser of that and $4,500 for single filers. That made 85% ($22,182) taxable instead of $18,542.70, so AGI came out at $54,076 instead of $50,437. It also put the 10% bracket ceiling at $12,150 rather than the 2026 figure of $12,400."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It assumed qualifying surviving spouse status without a dependent child, which inflated the deduction to about $40,000. It also overstated taxable SS at the full 85%, then declared taxable income 'near zero' even though its own figures left about $14,000. Single filing gives $26,287 of taxable income and $2,906.45 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It overstated taxable Social Security at the full 85% ($22,182) because it skipped the $4,500 single-tier cap; the correct amount is $18,542.70. It then computed about $3,300 of tax and replaced it with an unsupported $1,176. The correct path is $26,287 of taxable income, which produces $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It filed as a qualifying surviving spouse and used the $32,200 joint standard deduction, but that status requires a dependent child, and none is listed. Single filing uses $16,100 plus $2,050 aged plus $6,000 senior. Taxable income is therefore $26,287, not $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It used qualifying surviving spouse status (no dependent child qualifies) with a joint standard deduction of about $31,550, and it took 85% of benefits as taxable SS. It also left out the $6,000 OBBBA senior deduction. Single status gives taxable SS of $18,542.70, total deductions of $24,150, and $26,287 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It applied qualifying surviving spouse status, using MFJ thresholds and a $33,850 deduction without a dependent child. It counted 85% of SS as taxable, omitted the $6,000 senior deduction, and then arbitrarily raised its own $2,023 result to $3,204. Single filing gives $26,287 of taxable income and $2,906.45 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used the $32,200 joint standard deduction for surviving-spouse status, which requires a dependent child. With the correct single $18,150 age-adjusted standard deduction plus the $6,000 senior deduction, taxable income is $26,287 rather than $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its AGI of $50,436 was right, but it used an outdated $15,400 single base deduction instead of $16,100 and omitted the $6,000 OBBBA senior deduction. That left $32,986 taxable instead of $26,287. It also set the 10% bracket ceiling at $12,025 instead of $12,400."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It counted the full 85% of Social Security ($22,182) as taxable instead of $18,542.70 under the single $4,500-tier formula. It also omitted the $6,000 senior deduction and used a $17,350 standard deduction. Together these produced $36,726 of taxable income instead of $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law, with a $5,300 personal exemption and a 15% second bracket, even though OBBBA made the TCJA structure permanent. It also counted ESI premiums as a medical deduction. Under 2026 law the filer takes the $18,150 standard deduction plus the $6,000 senior deduction, and $26,287 is taxed at 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It got AGI ($50,436) and the $18,150 single age-adjusted standard deduction right but omitted the $6,000 OBBBA senior deduction for filers 65 and older. That left $32,286 taxable instead of $26,287. It also used a $12,200 10% bracket ceiling instead of $12,400."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, applying a personal exemption, a 15% bracket, and itemizing with ESI premiums treated as medical expenses. OBBBA kept the TCJA rate and deduction structure. The correct computation uses the $18,150 standard deduction plus the $6,000 senior deduction and the 10%/12% brackets on $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $37,990 taxable income leaves out the $6,000 senior deduction and understates the age-adjusted standard deduction, and its $1,109 tax does not follow from that taxable income under any bracket schedule. AGI of $50,437 less $24,150 of deductions gives $26,287 taxable and $2,906.45 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the expired pre-TCJA regime: a roughly $5,400 personal exemption, itemized deductions, and a 15% bracket. OBBBA made TCJA permanent, so there is no personal exemption, the filer takes the $18,150 standard deduction plus the $6,000 senior deduction, and the brackets are 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA reverted in 2026 (a $5,050 personal exemption, itemizing $16,470, and a 15% bracket above $11,000). Under OBBBA-permanent law the $18,150 standard deduction plus the $6,000 senior deduction exceed itemizing. That gives $26,287 of taxable income at 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that income falls below the standard deduction without computing taxable Social Security. AGI is $50,437 including $18,542.70 of taxable SS, and after $24,150 of standard, aged, and senior deductions, $26,287 remains taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the expired pre-TCJA structure: itemized property taxes plus a personal exemption, taxed in 10%/15% brackets. Under 2026 OBBBA law there is no personal exemption, and the $18,150 standard deduction plus the $6,000 senior deduction yield $26,287 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It itemized property taxes, took a personal exemption, and applied the pre-TCJA 15% bracket. OBBBA made TCJA permanent, so the single filer takes the $18,150 standard deduction plus the $6,000 senior deduction and pays 10%/12% on $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $50,436 was correct, but it used a $17,000 standard deduction instead of $18,150 ($16,100 plus $2,050 aged) and omitted the $6,000 senior deduction. That left $33,436 taxable instead of $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly filed as single with $50,436 AGI but omitted the $6,000 OBBBA senior deduction and used a $17,510 estimate instead of the $18,150 age-adjusted standard deduction. That left $32,926 taxable instead of $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $6,000 senior deduction, which applies whether or not the filer itemizes. It also treated the $8,389 ESI premiums as a deductible medical expense, which pushed itemized deductions to $21,076. The correct itemized total is $17,587.20, so the $18,150 standard deduction wins and taxable income is $26,287."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It relied on a 'surviving-spouse rate structure' that does not apply without a dependent child and claimed the deductions fully offset income. Single-status AGI of $50,437 less $24,150 of deductions leaves $26,287 taxable and $2,906.45 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the age-65 standard deduction wipes out all income, ignoring that $18,542.70 of Social Security is taxable and that AGI reaches $50,437. The total deductions of $24,150 leave $26,287 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint thresholds, so taxable SS came out at $11,542, along with a joint $32,200 standard deduction, but the head has no dependent child and files single. Single thresholds make $18,542.70 of SS taxable, and the $24,150 of deductions leave $26,287 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the right single-filer structure but miscalculated taxable Social Security as $18,592 instead of $18,542.70. That pushed AGI to $50,486 and taxable income to $26,336 instead of $26,287.11, about $49 too high, so it overshot the tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It filed as a surviving spouse with joint thresholds and a joint standard deduction even though there is no dependent child. That left only about $3,590 taxable instead of the $26,287 that results under single status."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It applied a $33,850 surviving-spouse (joint) age-adjusted standard deduction, but without a dependent child the head files single. The correct deductions are $18,150 plus the $6,000 senior deduction, giving $26,287 of taxable income rather than $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It followed the correct single path but overstated taxable Social Security by about $49. That carried through to taxable income of $26,336 instead of $26,287.11, so it reported $2,912 instead of $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint SS thresholds (taxable SS $11,542) and a $33,850 joint deduction, but qualifying surviving spouse status requires a dependent child. Single status gives $18,542.70 of taxable SS and $26,287 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard and senior deductions exceed taxable income. In fact AGI is $50,437 and total deductions are $24,150 ($16,100 + $2,050 + $6,000), which leaves $26,287 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (a $5,300 personal exemption, a reverted $16,600 standard deduction, and a 15% bracket) and used joint bracket widths for surviving-spouse status without a dependent child. Under 2026 OBBBA single rules, the $18,150 standard deduction plus the $6,000 senior deduction give $26,287 taxed at 10% up to $12,400 and 12% above."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired, restoring the personal exemption and the 15% bracket, and itemized property taxes. Under OBBBA-permanent 2026 law the $18,150 standard deduction plus the $6,000 senior deduction apply, giving $26,287 taxed at 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its filing status, AGI, and deductions were all correct ($26,286 taxable), but it put the 2026 single 10% bracket ceiling at $12,200 instead of $12,400. That shifted $200 from the 10% to the 12% rate and added about $4 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (uncapped SALT and a 15% bracket) and treated ESI premiums as an itemizable medical expense. It also omitted the $6,000 senior deduction. Under 2026 OBBBA law the $18,150 standard deduction plus the $6,000 senior deduction give $26,287 taxed at 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output. The correct derivation is single status, $50,437 of AGI, $24,150 of deductions, and $26,287 taxable, which gives $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It subtracted a $750 elderly credit and invented arithmetic to reach $0. The elderly credit is fully phased out because AGI of $50,437 far exceeds the $7,500 single threshold, and no other credit applies, so the tax on $26,287 of taxable income stands at $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It left half of Social Security out of provisional income (using $37,472 instead of $50,520) and dropped the $4,500 first-tier amount. That made only $2,951 of SS taxable instead of $18,542.70 and understated AGI by about $15,600. Its single filing status and $24,150 of deductions were correct."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed as a qualifying widow(er) with a joint $31,150 deduction although there is no dependent child. It also declared Social Security nontaxable even though provisional income of $50,520 exceeds every base amount. Single status makes $18,542.70 of SS taxable and leaves $26,287 taxable after the $24,150 of deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the single 65+ standard deduction eliminates taxable income. AGI is $50,437 including taxable Social Security, and the $18,150 standard deduction plus the $6,000 senior deduction leave $26,287 taxable, which owes $2,906.45."
us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 77 and an asserted income exclusion calculation as sufficient for Texas aged Medicaid without identifying or satisfying an aged eligibility pathway. The person qualifies through no Medicaid category and receives no SSI, so the claimed below-threshold aged eligibility does not apply."
us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were reported and that payroll tax therefore equals zero, but then submitted $1,534 instead. Its numeric output contradicts its own payroll-tax-base analysis and the required final-value contract."
@@ -84,55 +85,47 @@ us,scenario_002,federal_income_tax_before_refundable_credits,inkling,prompt_ambi
us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"After correctly dropping the disability and veterans benefits, it declared the full $44,080 of Social Security ""fully taxable since combined income exceeds the threshold"" — §86 never includes more than 85%, and at a provisional income of $40,780 (between the $32,000 and $44,000 MFJ thresholds) the includible amount is 50% × $8,780 = $4,390. Substituting that gives AGI of $23,130, below its own $32,600 deduction estimate and far below the actual $35,500 standard deduction plus $12,000 senior deduction, so tax is $0 instead of $3,149.40."
us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified a zero taxable wage base and then contradicted that computation by inventing taxable wage-equivalent income and assigning $1,926 of Social Security and Medicare taxes. No listed income is employee compensation subject to payroll tax, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components all equal zero."
us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly evaluated the spouse as a separate zero-income individual and assumed that age alone opened an aged Medicaid pathway. Washington's pathway evaluation includes the applicable household income and categorical requirements; this spouse receives no SSI, has household MAGI of 2.90 times FPL, and qualifies for no category."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"The model's own work reached about $22,379, using an incorrect $32,300 standard deduction and no $1,118 IRA deduction. It then submitted $27,049, which nothing in its reasoning supports."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $500 auto-loan interest deduction, although no facts show a qualifying new vehicle assembled in the U.S. It also left out the $1,118 traditional IRA deduction, which the frozen reference takes above the line and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The net result was taxable income of $150,590.50, against the frozen reference's $149,972.27, which gave $22,290.71, $109.94 below the corrected value, $22,400.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed only 85% of Social Security above $32,000 ($23,384) instead of the 85% cap of $50,583. It deducted a $23,926 401(k) deferral even though there are no wages. It also used a $28,900 standard deduction with an age-65 addition that neither spouse qualifies for, instead of the $32,200 MFJ amount, and then subtracted credits that do not exist."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It worked out about $22,652–$22,674 using 2025 bracket thresholds ($24,150/$98,200) and no IRA deduction. It then submitted $19,524 'given estimation uncertainty', a number that matches none of its calculations."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 bracket thresholds ($23,850/$96,950) instead of the 2026 MFJ thresholds ($24,800/$100,800). It also used a $32,300 standard deduction instead of $32,200 and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The 2025 thresholds overstate ordinary tax, so its $22,783 is $382.35 above the corrected value, $22,400.65 ($628.30 above the frozen reference)."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It reported ordinary tax of about $23,886 on $147,331 and cited a 24% bracket, but that income sits entirely inside the 22% bracket under the 2026 thresholds and the tax is about $21,837. This overstated the result by about $2,050. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA contribution out of above-the-line deductions because there is no compensation, while the reference subtracts it. That kept AGI at $183,290.50, where the frozen reference has $182,172.27. Every other step matches the reference, and $1,118 at the 22% marginal rate accounts for the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 standard deduction instead of the 2026 MFJ $32,200, and extrapolated bracket thresholds ($24,450/$99,375) instead of $24,800/$100,800. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so only the smaller standard deduction overstated its $153,291 of taxable income."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the $1,118 IRA deduction, as the frozen reference does, although the exclusion's corrected value disallows it because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It also overstated pre-Social Security income by $100 ($132,807 instead of $132,707) and used rough bracket thresholds of $24,000 and $97,000. It then taxed qualified dividends by subtracting the full 15% × $3,760 instead of the 7-point gap between the 22% and 15% rates, and rounded the result down to $22,000."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $183,291 subtracts only the $1,582 capital loss and leaves out the $1,118 traditional IRA contribution that the reference takes above the line. With everything else matching, the $1,118 at 22% explains the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It deducted a $23,500 traditional 401(k) deferral even though there are no wages to defer from; in the reference, 401(k) deferrals only reduce wages and give $0 here. It also used a $30,600 standard deduction instead of $32,200. Together these cut taxable income by about $22,000."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026, with personal exemptions, a $15,000 standard deduction and 10/15/25% brackets, even though the OBBBA made the TCJA rate structure and the $32,200 MFJ standard deduction permanent. It also made an AGI arithmetic error, getting $180,611."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly left Social Security out of income on the claim that benefits are exempt in 2026. With provisional income far above $44,000, 85% ($50,583) is taxable. It also used a $30,700 standard deduction instead of $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 MFJ standard deduction instead of the 2026 $32,200, and 2025 bracket thresholds ($23,850/$96,950) instead of $24,800/$100,800. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)); its taxable income was $153,290.50."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $16,000 standard deduction, $10,000 in personal exemptions and 10/15/25% brackets. Under the OBBBA, 2026 keeps the $32,200 MFJ standard deduction, no exemptions and the 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It treated about $183,865, which is roughly AGI, as taxable income and never actually subtracted the $32,200 standard deduction. It also skipped the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). This overstated taxable income by more than $33,000 against the frozen reference and more than $32,000 against the corrected value."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and used a pre-TCJA standard deduction plus personal exemptions, with 10/15/25% brackets, to reach taxable income of $155,490. The 2026 law uses the $32,200 standard deduction and the 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-sunset parameters: a $16,100 standard deduction, $10,100 in personal exemptions and pre-TCJA brackets. The 2026 law keeps the $32,200 MFJ standard deduction and the 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 tax without any calculation, even though taxable income is about $150,000 after the $32,200 standard deduction. That income produces about $21,591 of ordinary tax plus $564 on qualified dividends in the frozen reference, which subtracts the $1,118 IRA deduction; the exclusion's corrected value disallows that deduction because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), for about $21,837 of ordinary tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,800 standard deduction instead of $32,200. Its $23,112.71 result also shows it applied 2025 bracket thresholds ($23,850/$96,950) instead of 2026's $24,800/$100,800, and it left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied personal exemptions and a pre-TCJA standard deduction and brackets for 2026, even though the OBBBA made the TCJA structure permanent: no exemptions, a $32,200 MFJ standard deduction and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $23,287 matches a $30,000 standard deduction and 2025 bracket thresholds ($23,850/$96,950), instead of the 2026 $32,200 deduction and $24,800/$100,800 thresholds. It also made a $10 AGI arithmetic slip and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It misapplied the capital loss by using the $3,000 cap instead of the actual $1,582 net loss, and used a $30,000 standard deduction. It invented a $500 nonrefundable credit, and after getting $28,867 it submitted $49,523.45, which its reasoning does not support."
-us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain twice: once as income and again inside the net $1,582 loss. That inflated AGI to $184,370.50. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It then used a $32,600 standard deduction instead of $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed age-65 deductions even though the head is 64 and the spouse is 61, and it counted itemized medical expenses. The $10,900 in medical costs is below the 7.5%-of-AGI floor (about $13,660), so those deductions are $0 and taxable income stays near $150,000."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It deducted retirement contributions and itemized medical expenses that yield little or no deduction: 401(k) deferrals need wages, and medical costs fall below the 7.5%-of-AGI floor. That shrank taxable income far below the frozen reference's $149,972 (which the exclusion's corrected value raises by disallowing the $1,118 IRA deduction, because neither spouse has compensation) and produced only $2,320."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $151,091 comes from AGI of $183,291 with no $1,118 traditional IRA deduction, which the reference subtracts above the line. Everything else matches, and the IRA amount at 22% explains the $246 difference from the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA contribution out of above-the-line deductions, while the reference subtracts it. That kept AGI at $183,290.50, where the frozen reference has $182,172.27, and put tax $246 above the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $151,091 leaves out the $1,118 traditional IRA deduction that the reference takes above the line, where the reference gets $149,972.27. That produces the $246 gap to the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It reached taxable income of $151,090.50 without subtracting the $1,118 traditional IRA contribution that the reference deducts above the line. The remaining steps match, and the gap to the frozen reference is 22% × $1,118. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly treated the traditional IRA contribution as non-deductible because there are no wages, while the reference subtracts the $1,118 above the line. That left AGI at $183,290.50, where the frozen reference has $182,172.27, and tax $246 above the frozen reference. The exclusion's corrected value, $22,400.65, also treats the contribution as non-deductible because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $151,091 leaves out the $1,118 traditional IRA deduction that the reference applies above the line. Every other step matches, and the difference from the frozen reference is $1,118 taxed at 22%. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $183,290.50 with no traditional IRA deduction, while the reference subtracts the $1,118 contribution along with the capital loss. That put tax $246 above the frozen reference at the 22% marginal rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA deduction out of AGI, getting $183,290.50 where the reference gets $182,172.27, which made ordinary taxable income $147,330.50 instead of $146,212.27. That adds $246 at the 22% rate against the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and contributions wipe out all tax, but 401(k) deferrals need wages, and the only deductions available are the $1,582 capital loss and the $32,200 standard deduction; the frozen reference also deducts the $1,118 IRA contribution, which the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). That still leaves about $150,000 of taxable income."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: a $16,100 standard deduction, $10,600 in exemptions and 10/15/25% brackets. The OBBBA kept the $32,200 MFJ standard deduction and the 10/12/22% brackets for 2026."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a post-TCJA-sunset standard deduction plus personal exemptions and 10/15/25% brackets to get taxable income of $155,391. For 2026 the law uses the $32,200 standard deduction, no exemptions and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset and applied a $17,000 standard deduction, $10,800 in personal exemptions and pre-TCJA brackets. The 2026 law keeps the $32,200 MFJ standard deduction and the 10/12/22% brackets with no exemptions."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-TCJA 2026 parameters: a $17,103 standard deduction, $10,908 in exemptions and pre-TCJA brackets. These do not apply under the OBBBA, which sets the $32,200 MFJ standard deduction and the 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain twice, both as income and inside the $1,582 net loss, which inflated AGI to $184,371 and taxable income to $152,171, where the frozen reference has $149,972.27. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied 'current-law' post-sunset parameters: a $16,950 standard deduction, $10,800 in exemptions and 10/15/25% brackets. The OBBBA made permanent the $32,200 MFJ standard deduction and the 10/12/22% brackets for 2026."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used wrong 2026 bracket thresholds ($24,400/$99,150 instead of $24,800/$100,800). It also subtracted a $500 car-loan interest deduction with no qualifying vehicle facts and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It gave up because there are no wages and reported $0. Wages are not needed here: the retirement distributions, pension, taxable Social Security and investment income produce about $150,000 of taxable income and about $22,000 of tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $1,582 net capital loss twice, once through the ST/LT netting and again as a separate deduction, which dropped taxable income to $149,508.50. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so its taxable income was $464 below the frozen reference's and the whole gap to the corrected value, $22,400.65, comes from the second $1,582 subtraction."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $1,582 capital loss and overstated ordinary tax at $24,968. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It then submitted $41,228.50, which does not match its own $25,531.98 total."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $21,902.90 implies taxable income of about $148,828, roughly $1,145 below the reference's $149,972.27. That means it subtracted a deduction beyond the $1,582 capital loss and the $32,200 standard deduction, and beyond the $1,118 IRA contribution that the frozen reference also subtracts and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))."
-us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,The model replaced current-year Medicare eligibility with an invented near-term eligibility rule. Being age 64 and approaching 65 does not satisfy the age-65 threshold during a year in which age is constant.
-us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The reasoning correctly stated that the age-64 head is not Medicare eligible, but the model submitted value = 1, reversing its own eligibility conclusion. The required numeric encoding was 0 for “not eligible.”"
-us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a projection from age 64 to future age-65 eligibility. The benchmark holds age constant throughout 2026, and no disability or qualifying disease pathway is present, so the head fails every Medicare eligibility pathway."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working reached about $22,379 using a $32,300 standard deduction and the correct 2026 brackets. It then submitted $27,049, a number that nothing in its reasoning supports."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took a $500 deduction for auto loan interest. The facts never show a loan on a qualifying new, US-assembled vehicle, so the OBBBA car-loan interest deduction does not apply. That $500 at 22% is exactly the $110 shortfall from $22,400.65."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It taxed only 85% of the excess over $32,000 ($23,384) of Social Security instead of the 85% cap ($50,583). It also deducted the desired traditional 401(k) and IRA contributions even though no one has compensation. Finally, it treated a spouse as 65 or older and used an outdated $28,900 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It showed no calculation and was unsure about age-65 status. Neither spouse is 65, so no extra deduction applies. The correct taxable income of $151,090 produces $22,401. Its $14,800 implies about $34,500 less taxable income at 22%, consistent with adding retirement-contribution or age-65 deductions that are not available."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It calculated about $22,652–$22,674, even with 2025-style bracket limits ($24,150/$98,200). It then submitted an unrelated $19,524, citing estimation uncertainty instead of reporting its own result."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used 2025 bracket limits (10% to $23,850, 12% to $96,950) and a $32,300 standard deduction. The 2026 values are $24,800, $100,800 and $32,200. This pushed more ordinary income into the 22% bracket and overstated tax by about $382."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It applied a 24% bracket to ordinary taxable income of about $147,331. The 2026 joint 22% bracket runs to about $211,400, so ordinary tax is $21,837, not $23,886."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 joint standard deduction as $30,000 by indexing pre-OBBBA amounts. OBBBA set it at $32,200. It also used projected bracket limits ($24,450/$99,375) instead of $24,800/$100,800, which overstated taxable income and tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the $1,118 traditional IRA contribution even though neither spouse has compensation, so the deductible limit is $0. It also overstated pre-Social Security income by $100 ($132,807) and rounded its approximate result down to $22,000."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted a $23,500 traditional 401(k) deferral and a $1,118 IRA deduction even though the household has no wages or other compensation, so neither is deductible. It also used a $30,600 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and applied a $15,000 standard deduction, $5,100 personal exemptions and the 10/15/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, no exemptions and the 10/12/22% brackets. It also understated AGI at $180,611."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It wrongly claimed Social Security is excluded from income in 2026. Section 86 still applies, and with provisional income near $167,704, $50,583 is taxable. It also used a $30,700 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and 2025 bracket limits (10% to $23,850, 12% to $96,950). The 2026 values are $32,200 and $24,800/$100,800, so it overstated ordinary tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the TCJA sunset rules: a $16,000 standard deduction, $10,000 of personal exemptions and the 10/15/25% brackets. Under OBBBA, 2026 uses a $32,200 standard deduction, no exemptions and the 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $183,865 is higher than AGI, so it never subtracted the $32,200 standard deduction. The correct taxable income is $151,090."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and subtracted a pre-TCJA standard deduction plus personal exemptions, reaching about $155,490 of taxable income taxed at the old rates. OBBBA kept the $32,200 standard deduction, no exemptions and the 10/12/22% brackets for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the TCJA sunset rules: a $16,100 standard deduction, $10,100 of personal exemptions and the higher pre-TCJA brackets. OBBBA made the $32,200 standard deduction and the 10/12/22% brackets permanent for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It reported $0 with no calculation, even though $151,090 of taxable income remains after the $32,200 standard deduction. That income produces $21,837 of ordinary tax plus $564 on qualified dividends."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and 2025 bracket limits (10% to $23,850, 12% to $96,950). The 2026 values are $32,200 and $24,800/$100,800, which explains its $23,112.71."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted personal exemptions and a pre-TCJA standard deduction for 2026, assuming the TCJA had expired. OBBBA eliminated exemptions permanently and set the joint standard deduction at $32,200 with the 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $23,287 matches a $30,000 standard deduction and 2025 bracket limits ($23,850/$96,950) applied to a slightly misstated AGI of $183,280.50. The 2026 values are a $32,200 standard deduction and $24,800/$100,800 bracket limits."
+us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Its own working ended at $28,867, which already had several errors: it capped the long-term loss at -$3,000 instead of netting to -$1,582, used a $30,000 standard deduction, taxed qualified dividends twice, and invented a $500 'sick spouse' credit. It then submitted an unrelated $49,523.45."
+us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain as separate income and also inside the -$1,582 net capital loss, overstating AGI at $184,370.50 instead of $183,290.25. It also used a $32,600 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It claimed age-65 deductions, but the head is 64 and the spouse 61, so no additional or senior deduction applies. Medical expenses also fall below the 7.5%-of-AGI floor (about $13,747), so itemizing gives nothing. The $32,200 standard deduction leaves $151,090 taxable, not $0."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It deducted retirement contributions, which are not allowed without compensation, and itemized medical expenses that fall below the 7.5%-of-AGI floor. Its $2,320 implies only about $23,000 of taxable income, when $151,090 remains after the $32,200 standard deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It said the desired 401(k)/IRA contributions and the standard deduction wipe out tax. With no compensation those contributions are not deductible, and the $32,200 standard deduction leaves $151,090 taxable, producing $22,400.65."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the TCJA sunset rules: a $16,100 standard deduction, $10,600 of personal exemptions and the 10/15/25% brackets. OBBBA made the $32,200 standard deduction, zero exemptions and the 10/12/22% brackets permanent for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used post-sunset pre-TCJA rules (standard deduction plus personal exemptions, 10/15/25% brackets) to reach $155,391 of taxable income. Under OBBBA, 2026 taxable income is $151,090 after the $32,200 standard deduction and is taxed at 10/12/22%."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, using a $17,000 standard deduction, $10,800 of exemptions and pre-TCJA brackets. OBBBA kept the $32,200 joint standard deduction with no exemptions and the 10/12/22% brackets for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used inflation-adjusted pre-TCJA parameters: a $17,103 standard deduction, $10,908 of exemptions and the 10/15/25% brackets. 2026 law under OBBBA uses a $32,200 standard deduction, no exemptions and the 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $1,080 short-term gain as income and then also subtracted the -$1,582 net capital loss that already includes it. That overstated AGI at about $184,371 instead of $183,290 and pushed the tax above $22,400.65."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used projected pre-TCJA 2026 law: a $16,950 standard deduction, $10,800 of exemptions and the 10/15/25% brackets. OBBBA made permanent the $32,200 standard deduction, no exemptions and the 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used bracket limits of $24,400 and $99,150 instead of 2026's $24,800 and $100,800, which overstated ordinary tax. It also took a $500 car-loan-interest deduction that the facts don't support, since nothing shows a loan on a qualifying new, US-assembled vehicle."
+us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 because there were no wages and never taxed the non-wage income: $94,720 of IRA distributions, $32,525 of pension, $50,583 of taxable Social Security, and interest and dividends. That income produces $151,090 of taxable income."
+us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,582 net capital loss twice. AGI was already $183,290.50, but it reported taxable income of $149,508.50 instead of $151,090.50, understating tax by $348 (22% of $1,582)."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its working reached $25,531.98 after ignoring the $1,582 capital loss, treating the head as 65 or older, and miscalculating ordinary tax. It then submitted an unrelated $41,228.50."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The correct path is AGI of $183,290.25 minus the $32,200 standard deduction, giving $151,090.25 taxable and $22,400.65 of tax. Its $21,902.90 matches about $2,260 less taxable income at the 22% rate, an unexplained extra reduction that the facts do not support."
+us,scenario_003,head_medicare_eligible,claude-haiku-5.5,llm_error,other,False,"The model's explanation correctly said the head, at 64, is under the age-65 Medicare threshold and has no disability pathway, and it ended with ""value = 0"". Its submitted numeric value was still 1 (Yes), so the output field contradicts its own correct reasoning."
+us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented a ""near-term eligibility"" rule that counts a 64-year-old as Medicare-eligible. Eligibility requires reaching age 65 (or a disability/ESRD pathway, none of which is present). The prompt also fixes age as constant for the tax year, so no projection to 65 applies."
+us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly said that 64 is below the Medicare eligibility age of 65 and that the head is ""not eligible"". It then submitted value = 1, which contradicts its own conclusion; the correct value is 0."
+us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model assumed that age-65 Medicare eligibility is ""projected"" onto a 64-year-old for the modeled year. No such rule exists: the age test is applied to the head's actual age, 64, which stays constant all year. With no disability or ESRD, the head is not eligible."
us,scenario_003,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model invented a $10 hourly wage for both adults from their weekly hours even though hourly wage and wage income were unlisted and therefore zero under the prompt. It then applied Social Security and Medicare rates to fabricated annual wages, and its stated total also adds an unexplained $947.17 beyond its own $1,354.08 and $316.68 components."
us,scenario_003,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model annualized weekly hours into labor earnings without any listed hourly wage or wage amount. Weekly hours alone create no payroll-tax base; with unlisted wages set to zero, Social Security and Medicare employee taxes are both zero."
us,scenario_003,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no general individual income tax, then invented a $280 “nonstandard component” unsupported by any Texas tax rule or household fact. It failed to carry the Texas zero through the 51-jurisdiction aggregation, whose components all equal zero because the household resides exclusively in Texas."
@@ -142,10 +135,11 @@ us,scenario_004,federal_income_tax_before_refundable_credits,gemini-3.1-flash-li
us,scenario_004,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, merely restating the number; $4,635 corresponds to roughly $42,700 of taxable income, consistent with including about 85% of the $51,475 Social Security benefit in gross income and applying essentially no standard deduction. Because provisional income of $26,802.50 is below the $32,000 MFJ base amount, none of the benefits is taxable, AGI is $1,065, and the 2026 MFJ standard deduction of $32,200 drives taxable income and tax to $0."
us,scenario_004,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It ran the §86 test correctly — provisional income $26,802.50 below the $32,000 MFJ threshold, so benefits are excluded — then contradicted its own conclusion by subtracting the standard deduction from the full $52,540 AGI that still contained the excluded $51,475 of Social Security, yielding $21,590 of taxable income and $2,159 of tax. With the benefits excluded, only $1,065 of income remains to absorb the standard deduction, so taxable income and tax before refundable credits are $0."
us,scenario_004,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as $1,065 by using $1,200 of taxable interest minus the $135 partnership loss and omitting the Social Security retirement income included in the Medicaid MAGI calculation. The resulting MAGI is approximately $52,540, or 2.39 times FPL, above the expansion-adult threshold, and the head qualifies through no other Medicaid category."
-us,scenario_004,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model acknowledged the age-65 rule but invented an unspecified special-status pathway despite all unlisted statuses being false. At age 62 with no qualifying disability, end-stage renal disease, or ALS status, the head is not Medicare eligible."
-us,scenario_004,head_medicare_eligible,gpt-5.4-nano,llm_error,other,False,The model's explanation computed Medicare eligibility as 0 under the age-65 rule but submitted value = 1. This is an internal answer-contract contradiction: its numeric output does not match its stated conclusion.
-us,scenario_004,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model incorrectly treated receipt of Social Security retirement benefits at age 62 as conferring Medicare eligibility. Retirement-benefit receipt does not replace the age-65 requirement, and no qualifying disability, end-stage renal disease, or ALS pathway is present."
-us,scenario_004,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model directly misapplied the age threshold by stating that age 62 meets a requirement of age 65 or older. Because 62 is below 65 and no non-age eligibility pathway is listed, the head is not Medicare eligible."
+us,scenario_004,head_medicare_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model correctly said age 62 alone is not enough, then wrongly treated the head's Social Security retirement income as a path to Medicare. Early retirement benefits do not create Medicare eligibility; under 65, only 24 months of disability benefits or ESRD/ALS do, and none applies to this head."
+us,scenario_004,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model knew the threshold is typically 65 but answered yes by pointing to vague 'specific statuses' and 'internal logic' without naming any real pathway. The head is 62 with no disability, SSDI, or ESRD, so the age-65 test fails and the answer is not eligible."
+us,scenario_004,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's own reasoning correctly applied the age-65 threshold and concluded 'Medicare eligible=0'. It then submitted value = 1, which contradicts that conclusion. The head is 62 with no disability pathway, so 0 is the answer its reasoning supported."
+us,scenario_004,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model claimed that receiving Social Security retirement benefits at 62 makes the head eligible for Medicare. Retirement benefits taken at 62 do not confer Medicare. Eligibility requires age 65, 24 months of SSDI, or ESRD/ALS, and the head meets none of these."
+us,scenario_004,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model stated that age 62 'meets' a threshold of 65 or older, which is an arithmetic misapplication of the age test. At 62, the head falls short of 65 and has no disability or ESRD pathway, so the head is not eligible."
us,scenario_004,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were listed and that Social Security retirement income does not generate employee payroll tax, but then submitted $5,108 instead of the resulting $0. Its numeric answer applies payroll tax to a nonexistent or ineligible earnings base and directly contradicts its own derivation."
us,scenario_004,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly assigned the spouse MAGI of $1,065 instead of using the tax-unit MAGI that places the spouse at 2.39 times FPL, evidently omitting the head's Social Security retirement and other relevant tax-unit income from the Medicaid MAGI computation. Because 2.39 times FPL exceeds the 138% expansion limit and the spouse qualifies through no categorical pathway, the spouse is not eligible."
us,scenario_004,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It botched the federal provisional-income test, declaring $38,606 of the $51,475 benefit taxable when provisional income is $26,803 ($1,065 of non-SS income plus half the benefit), below the $32,000 MFJ base amount, so $0 of Social Security is federally taxable — and New York additionally subtracts taxable Social Security from federal AGI in full. It then applied a $35,550 standard deduction rather than New York's ~$16,050 MFJ figure, and reported $1,380 on its own stated $4,121 taxable base, which produces $165 at NY's 4% bottom rate."
@@ -154,181 +148,185 @@ us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite
us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 implies about $64,000 of New York taxable income under the MFJ brackets — more than the household's entire $52,540 of gross receipts including Social Security — so it both taxed the $51,475 benefit and imputed unlisted wage earnings from the 40-hour work weeks that the prompt requires be treated as 0. The actual New York base is $1,065 of interest less partnership loss, fully offset by the ~$16,050 MFJ standard deduction, for $0 of tax."
us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrote that New York does not tax Social Security yet built its base from $52,540 with the full $51,475 benefit included, never applying the federal provisional-income test ($26,803 against the $32,000 MFJ base) that makes the benefit 0% taxable and therefore leaves only $1,065 of federal AGI to flow to New York. Its arithmetic is also self-contradictory: $52,540 less its stated $16,850 standard deduction is $35,690, not the $18,426 it claimed, and $3,451.12 on $18,426 is an 18.7% effective rate against NY's 4%–4.5% brackets at that level (about $743). The correct result is $1,065 of NY AGI wiped out by the ~$16,050 standard deduction, giving $0."
us,scenario_004,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model correctly found federal AGI under $5,000 and used the table amount of $90 plus $15 for the second exemption, but it counted the NY household credit (Tax Law §606(b)) as refundable. That credit is nonrefundable: it only offsets NY tax, which is zero here, and it never enters state_refundable_credits. None of NY's refundable credits apply (EITC, Empire State child credit, real property tax credit), so the answer is $0, not $105."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left the $5,722 net investment income tax (3.8% × $150,578) out of this variable. Used a $32,600 standard deduction instead of $32,200 plus the $1,070 non-itemizer cash-charity deduction. Its computed $100,206 was then raised by an arbitrary 'variation' to $100,819."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Left out the OBBBA non-itemizer cash-charity deduction ($1,070 of cash gifts on top of the $32,200 standard deduction). It also used full $46,308 traditional 401(k) deferrals instead of $41,650 (prorated to fit the $24,500 combined traditional+Roth limit) and included the $4,148 state refund. Net result: taxable income is $559 too high, which adds $179 at 32%."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $110,000 of mortgage interest from the $2.85M balance even though no interest paid was listed. It then reported $249,914, which follows from no bracket computation. The standard-deduction path gives the frozen reference's $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly excluded NIIT from this line, although the variable includes the $5,722 NIIT. The final $91,458 also drops the $11,688 qualified-dividend tax from its own $102,907 regular-tax total, leaving roughly the ordinary-bracket tax alone. It also used a $31,500 standard deduction and estimated brackets instead of the 2026 values ($32,200; 32% bracket starting at $403,550)."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Imputed mortgage interest from the $2.85M balance to reach about $60,000 of itemized deductions, even though no interest was listed. It taxed qualified dividends at 20% instead of 15%. Its $154,430 total also contradicts its own parts ($92,000 + $15,584 = $107,584)."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Summed its own AGI components to $604,318 instead of $538,418, an arithmetic error of about $66,000. It then subtracted about $60,000 of itemized deductions built on mortgage interest the facts never provide. Taxable income ended up at $544,000 instead of $503,495."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $1,070 non-itemizer cash-charity deduction that 2026 law adds to the $32,200 standard deduction. It also kept the full $46,308 of 401(k) deferrals rather than the $41,650 prorated under the $24,500 combined limit, and included the $4,148 refund. Taxable income was $504,054 instead of $503,495, overstating tax by $179."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired for 2026, but OBBBA made it permanent. It therefore applied inflated pre-TCJA brackets (up to 33%), a $10,000 SALT cap, and restored 2%-floor miscellaneous deductions. It also imputed $52,500 of mortgage interest at an assumed 7% rate and added a $7,141 AMT that does not arise under the 2026 AMT exemption."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $28,000 of mortgage interest from the loan balance and applied the old $10,000 SALT cap, then itemized $40,574. It asserted a rough $115,000 without a bracket computation. The correct path uses the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction. With NIIT the frozen reference totals $106,506; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted $2,574 of charitable gifts as itemized without applying the 2026 floor of 0.5% of AGI, which reduces them to zero. It also compared against the bare $32,200 standard deduction instead of $32,200 plus the $1,070 non-itemizer cash-charity deduction. Standard ($33,270) actually beats itemizing, since SALT is capped at about $30,900; this error, plus using full 401(k) deferrals, put its total $144 too high."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of the 2026 MFJ $32,200 plus the $1,070 non-itemizer cash-charity deduction. It computed NIIT on $153,578 without subtracting the allowable $3,000 capital loss, when the correct base is $150,578. It also kept full 401(k) deferrals and included the $4,148 refund."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Applied an AMT of $131,247 that exceeds regular tax. Under 2026 law the MFJ AMT exemption ($140,200) does not start phasing out until $1,000,000, so no AMT arises for a standard-deduction filer with $537k of AGI. Its $465,449 taxable income also reflects inflated itemized deductions instead of the $33,270 standard-plus-charity deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a Pease limit on itemized deductions and a pre-TCJA AMT that raised tax to $127,938. OBBBA made TCJA permanent, so 2026 has no Pease limit and no AMT at this income. It also computed NIIT on $153,578 instead of the $150,578 base that nets the $3,000 capital loss."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income using pre-TCJA sunset brackets for 2026 instead of the permanent 10/12/22/24/32/35/37% OBBBA brackets. It itemized $63,508 instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration, so it allowed uncapped SALT, imputed mortgage interest on a $1M balance even though no interest was listed, and claimed $10,400 of personal exemptions. None of these exist in 2026 law. Under the standard deduction the frozen reference leaves $503,495 taxable, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its near-miss total comes from offsetting errors."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Built AGI of $460,936 that counts the tax-exempt $70,000 pension as income while omitting large parts of the $87,642 of dividends and other investment income; the frozen reference's AGI is $536,765, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its $110,996 does not follow from 2026 brackets, the 15% qualified-dividend rate and the $5,722 NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not arise under 2026 law, where the $140,200 MFJ exemption phases out only above $1,000,000. It also cut AGI to about $525,049 by subtracting the $13,369 of employer insurance premiums from the listed gross wages."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA individual provisions expired in 2026 and deducted full, uncapped SALT with pre-TCJA brackets up to 33%. Under the permanent OBBBA rules, SALT is phased down to about $30,900 and the standard deduction plus the $1,070 charity deduction wins. The frozen reference's ordinary tax is then $89,096 at the 2026 brackets, on taxable income net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. The frozen reference's figure is $89,095.92 of ordinary tax on $425,575, plus $11,688 on $77,920 of qualified dividends, plus $5,721.98 NIIT; its taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)). The answer is $413 short of that, consistent with overstating deductions by about $1,290 at the 32% marginal rate."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from the listed gross wages, cutting AGI to $525,049. It then applied sunset-era itemized deductions and personal exemptions, and taxed qualified dividends at 20% instead of 15%. The 15% bracket extends to $613,700 for MFJ in 2026."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration brackets and itemized with imputed mortgage interest, even though no interest was listed. It also subtracted health premiums from wages. That drove the result to $88,392, far below the frozen reference's $106,506, which the 2026 permanent brackets, the $33,270 standard-plus-charity deduction and $5,722 NIIT produce together with a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used post-TCJA-sunset rules: itemizing state income tax without the phased-down SALT cap and restoring 2%-floor miscellaneous deductions. OBBBA permanently eliminated miscellaneous deductions, and the $33,270 standard-plus-charity deduction applies. It also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), and reported a rounded $105,000."
-us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Left the $5,722 NIIT out of this variable. It also used 2025 parameters (a $30,000 standard deduction and 2025 brackets) instead of the 2026 values ($32,200 plus the $1,070 non-itemizer charity deduction)."
-us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Took the full $2,000 cap for the non-itemizer charity deduction, although only the $1,070 of cash gifts qualifies and non-cash gifts are excluded. It kept full $46,308 401(k) deferrals. It disallowed the $2,164 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out. It also computed NIIT on $153,578 without netting the $3,000 capital loss."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735 with no computation. Taxable income of $503,495 under 2026 MFJ brackets yields $89,096 of ordinary tax alone, before $11,688 of qualified-dividend tax and $5,722 of NIIT. The answer omits most of the ordinary-income tax."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Reported $75,657 without showing brackets, and cited itemized medical deductions that are zero under the 7.5%-of-AGI floor. The frozen reference's total is $89,096 of ordinary tax, $11,688 of qualified-dividend tax and $5,722 of NIIT; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)). The answer is about $30,800 short, missing roughly the qualified-dividend tax, the NIIT and part of the 32%-bracket tax."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used AGI of $538,418 instead of $536,765 for three reasons. It deducted the full $46,308 of traditional 401(k) deferrals instead of $41,650 prorated under the $24,500 combined limit. It included the $4,148 state refund, which the reference excludes. It disallowed the $2,164 traditional IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out. The resulting $1,653 of taxable income above the frozen reference's, taxed at 32%, explains its $529 excess over the frozen reference; the answer is $163.42 below the corrected value."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no figures. The frozen reference reaches $503,495 of taxable income after the $32,200 standard deduction and $1,070 charity deduction, with tax of $89,096 + $11,688 + $5,722; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its $104,535 is $1,971 low, consistent with understating taxable income by about $6,200 at the 32% marginal rate."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 state refund in AGI, kept full 401(k) deferrals, and itemized instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer cash-charity deduction. That overstated taxable income by about $2,000, which adds $632 at 32%."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $32,200 standard deduction and left out the $1,070 non-itemizer cash-charity deduction. It also computed AGI as $538,418 by including the $4,148 refund, deducting the full $46,308 of 401(k) deferrals instead of $41,650, and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.66 above the corrected value."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer health premiums from the listed gross wages, reaching AGI of $525,049 instead of $536,765. It then itemized $36,959, but the charitable portion is zeroed by the 0.5%-of-AGI floor and SALT is phased down to about $30,900. The correct deduction is the $33,270 standard-plus-charity amount."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly included the $4,148 state refund in AGI, which the reference excludes, and deducted the full $46,308 of 401(k) deferrals instead of the prorated $41,650. It also took the plain standard deduction without the $1,070 non-itemizer cash-charity deduction. Together these overstated taxable income and pushed tax $571 above the frozen reference's total; the exclusion's corrected value, $107,198.34, which disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $121.34 above the answer."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from the listed gross wages to reach AGI of $525,049. It then itemized a phased-down SALT deduction instead of taking the larger $32,200 standard deduction plus the $1,070 non-itemizer charity deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Reduced AGI to $525,049 by subtracting the $13,369 of employer insurance premiums from the listed gross wages; the frozen reference's AGI is $536,765 on wages of $388,350, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). It then itemized $34,385 instead of the $33,270 standard-plus-charity deduction, so taxable income came out $12,831 too low."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Invoked a qualified business income deduction for a household with no business income and gave no bracket computation. Its $112,450 is $5,944 above the $89,096 + $11,688 + $5,722 total that 2026 brackets produce on $503,495 of taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: Pease, personal exemptions, miscellaneous deductions, and an AMT that exceeds regular tax. OBBBA made TCJA permanent, so none of these apply in 2026. The frozen reference's result is regular tax on $503,495 plus NIIT with no AMT; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Used 2026 sunset law, with Pease-limited itemized deductions and a $25,486 AMT driven by SALT and miscellaneous add-backs. Under permanent TCJA/OBBBA rules miscellaneous deductions do not exist, the standard deduction applies, and the $140,200 AMT exemption leaves no AMT at this income."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of the 2026 $32,200 and left out the $1,070 non-itemizer cash-charity deduction. It also computed AGI as $538,418 by including the $4,148 refund, deducting the full $46,308 of 401(k) deferrals and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Together these put taxable income $3,423 above the frozen reference's, and the answer is $332.66 above the corrected value."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Computed $106,958 of ordinary tax on $423,928, which reflects pre-TCJA-style rates; the 2026 brackets give about $88,600. It also subtracted $13,369 of employer premiums from wages and itemized with uncapped property tax and 2%-floor miscellaneous deductions, which OBBBA permanently repealed."
-us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Applied sunset-era rules: $64,011 of SALT, miscellaneous deductions, a Pease reduction, and an $18,221 AMT. None of these apply under the permanent 2026 rules, where SALT is phased down to about $30,900, the standard-plus-charity deduction wins, and there is no AMT. It also subtracted employer premiums from wages."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Took the $32,200 standard deduction but left out the 2026 non-itemizer deduction for the $1,070 of cash charitable gifts. It also used AGI of $538,418 by including the $4,148 refund, deducting full $46,308 401(k) deferrals instead of $41,650, and disallowing the $2,164 IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.98 above the corrected value."
-us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that itemized deductions, including mortgage interest never listed in the facts, wipe out all tax. In fact no interest was paid, SALT is capped, and the standard-plus-charity deduction of $33,270 leaves $503,495 taxable in the frozen reference, which produces $106,506 of tax including NIIT; the exclusion's corrected value, $107,198.34, also disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $1,070 non-itemizer cash-charity deduction and used AGI of $538,418, including the refund and full 401(k) deferrals. It miscomputed the NIIT base as $142,667; the correct base is $150,578 of interest plus dividends less the $3,000 capital loss. These errors partially offset to $107,077."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the full $10,911 net capital loss instead of the $3,000 limit and imputed $43,118 of mortgage interest from the balance. It netted the capital loss against qualified dividends, shrinking the 15% base to $67,009 instead of $77,920. It used wrong 2026 bracket edges and left out the $5,722 NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Used the $15,700 single-filer standard deduction for a joint return instead of the $32,200 MFJ deduction plus the $1,070 non-itemizer charity deduction. It added AMT, which does not apply, and left out the $5,722 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the 3.8% net investment income tax ($5,721.98 on $150,578.34 of NII), which this output includes; its ~$100.2k regular-tax figure was nudged to $100,819 and NIIT was never added. It also kept the full $46,308 of 401(k) exclusions instead of the $41,650 left after pro-rating to the $24,500 combined traditional+Roth limit, and it denied the $2,164 IRA deduction and the $1,070 non-itemizer charitable deduction."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as income (PolicyEngine excludes it when no prior-year itemization benefit exists), excluded the full $46,308 of 401(k) deferrals instead of $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. The errors partly offset, leaving taxable income of $504,054 versus $503,494.75 and tax $178.94 too high at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $110,000 of mortgage interest from the $2.85M balance even though no interest paid was listed, so mortgage interest is $0 and the $32,200 standard deduction plus $1,070 non-itemizer charity applies. Its $249,914 also does not follow from its own ~$408,000 taxable income; the correct path is $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Refused to compute the liability and entered a placeholder $0 despite $503,494.75 of taxable income. Ordinary bracket tax ($89,095.92), 15% tax on $77,920 of qualified dividends ($11,688), and NIIT ($5,721.98) add up to $106,505.90."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly excluded NIIT ($5,721.98) from this line, then reported $91,458, which also drops its own $11,688 qualified-dividend tax and roughly equals its ordinary-bracket tax alone. It also used outdated 2026 brackets (24% ending near $396,000 instead of $403,550) and the full $46,308 401(k) exclusion instead of the pro-rated $41,650."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Invented mortgage interest from the $2.85M balance even though no interest paid was listed, producing ~$60,000 of itemized deductions instead of the $32,200 standard deduction plus $1,070 non-itemizer charity. It also taxed qualified dividends at 20% even though ~$503k of taxable income falls in the 15% band, and its $154,430 exceeds the sum of its own components (~$92,000 + $15,584) by about $47,000."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Added AGI up to $604,318 when its own listed components sum to $538,418, and invented mortgage interest from the $2.85M balance (no interest paid was listed) to reach ~$60,000 of itemized deductions. It also used an 18.8% blended rate on qualified dividends instead of 15%, overshooting the correct $89,095.92 + $11,688 + $5,721.98."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as gross income (PolicyEngine excludes it), excluded the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. Its taxable income of $504,054 versus $503,494.75 overstated tax by $178.94 at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset law for 2026: pre-TCJA 15/25/28/33% brackets, restored 2%-floor miscellaneous deductions, and a pre-TCJA AMT exemption that produced $7,141 of AMT. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $140,200 MFJ AMT exemption (phase-out starting at $1,000,000) permanent. It also invented $52,500 of mortgage interest from a balance with no stated interest and computed NIIT on $153,578 without the $3,000 capital-loss offset."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Invented ~$28,000 of mortgage interest from the $2.85M balance even though no interest paid was listed, and itemized with a $10,000 SALT cap instead of taking the $32,200 standard deduction plus $1,070 non-itemizer charity. It never ran the brackets and guessed $115,000; even its own $497,844 taxable income yields about $104,700 including NIIT under 2026 brackets."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized the full $2.6k of charity on top of a ~$31.9k SALT deduction, ignoring the 0.5%-of-AGI floor that eliminates the itemized charitable deduction, so the $32,200 standard deduction plus $1,070 non-itemizer cash deduction is the right choice. It also counted the $4,148 refund, denied the $2,164 IRA deduction, and excluded the full $46,308 of 401(k) deferrals instead of $41,650; the offsetting errors left it $144 high."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of the 2026 OBBBA $32,200 plus the $1,070 non-itemizer cash charitable deduction, and got $92,234 of ordinary tax on $429,748 where 2026 brackets give $90,431. It also computed NIIT on $153,578 without netting the $3,000 allowable capital loss (correct NII is $150,578.34), counted the $4,148 refund, and used the unprorated $46,308 401(k) exclusion."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not bind: under OBBBA the 2026 MFJ AMT exemption is $140,200, phasing out only above $1,000,000 of AMTI, so tentative minimum tax on ~$537k of AMTI stays below the $100,783.92 regular tax. Its $465,449 taxable income also reflects itemized deductions far above the $33,270 standard-plus-charity deduction, and its $5,836 NIIT ignores the $3,000 capital-loss offset."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules, namely the Pease limit on itemized deductions and an AMT that raised tax to $127,938, but OBBBA permanently repealed Pease and set the 2026 AMT exemption at $140,200 with a $1,000,000 phase-out threshold, so no AMT arises. It also claimed ~$59.5k of itemized deductions with no listed mortgage interest instead of the $32,200 standard deduction plus $1,070 non-itemizer charity, and used $153,578 of NII without the $3,000 capital-loss offset."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income on pre-TCJA 'sunset' 2026 brackets and itemized $63,507.64, but OBBBA made the TCJA brackets permanent, and with no mortgage interest listed the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity. It also computed NIIT on $153,578 instead of $150,578.34 after the $3,000 capital loss."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expired, using uncapped SALT, mortgage interest invented on $1M of acquisition debt with no interest listed, and $10,400 of personal exemptions, for $108,283 of itemized deductions. Under OBBBA personal exemptions stay at zero, SALT is capped at a phased-down ~$30.9k, and the $32,200 standard deduction plus $1,070 non-itemizer charity applies. Its errors netted to $88,016 of ordinary tax, $1,080 below the correct $89,095.92."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Reported AGI of $460,936 'including pensions', but the $70,000 pension is tax-exempt and correct AGI is $536,764.50 (wages net of the pro-rated $41,650 of 401(k) deferrals, plus interest and dividends, less the $3,000 capital loss and $2,164 IRA deduction). It gave no bracket, qualified-dividend, or NIIT breakdown, and its $110,996 overshoots by $4,490."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not arise under OBBBA's 2026 exemption ($140,200 MFJ, phase-out starting at $1,000,000 of AMTI). It also reduced wages by the $13,369 of employee ESI premiums to reach AGI of $525,049, while PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals (AGI $536,764.50)."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Treated TCJA as expired in 2026, deducting uncapped SALT (state income plus real estate tax) and using brackets that top out at 33%. OBBBA permanently kept TCJA rates and caps SALT at a phased-down ~$30.9k, which loses to the $32,200 standard deduction plus $1,070 non-itemizer charity, so the answer is $89,095.92 + $11,688 + $5,721.98."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. The correct path is taxable income of $503,494.75 ($32,200 standard deduction plus $1,070 non-itemizer charity), $89,095.92 ordinary tax, $11,688 of 15% qualified-dividend tax, and $5,721.98 NIIT. Its $106,093 is $412.90 short, which equals understating taxable income by about $1,290 at the 32% marginal rate."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (personal exemptions plus itemized deductions), subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049 instead of $536,764.50, and taxed qualified dividends at 20% even though ~$503k of taxable income falls in the 15% band. The correct total is $89,095.92 ordinary tax + $11,688 + $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration brackets, treated health premiums as pre-tax wage reductions, and invented mortgage interest from the $2.85M balance even though no interest paid was listed, pushing tax down to $88,392. Under OBBBA's permanent rules the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity, giving $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA sunset rules, deducting uncapped state income tax and 2%-floor miscellaneous deductions for the $13,848 of unreimbursed employee expenses. Under OBBBA miscellaneous itemized deductions stay suspended, SALT is capped (phased down to ~$30.9k), and the $32,200 standard deduction plus $1,070 non-itemizer charity wins. It also used AGI of $534,270 (no IRA deduction, unprorated 401(k)) and rounded to $105,000."
+us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $5,721.98 net investment income tax, which this output includes. It also used 2025 parameters (the $30,000 standard deduction and 2025 brackets) instead of 2026's $32,200 standard deduction, the $1,070 non-itemizer cash charitable deduction, and inflation-adjusted brackets."
+us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Took the maximum $2,000 non-itemizer charitable deduction even though only $1,070 of cash gifts qualify (non-cash gifts are excluded), and excluded the full $46,308 of 401(k) deferrals instead of $41,650 pro-rated to the $24,500 combined limit. Its taxable income was $500,070 versus $503,494.75, and its NIIT was computed on $153,578 without the $3,000 capital-loss offset."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735, about one-fifth of the correct liability, with no bracket computation. On $503,494.75 of taxable income, ordinary bracket tax alone is $89,095.92, before the $11,688 qualified-dividend tax and $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Gave no numeric derivation and mixed standard and itemized deductions. Its $75,657 is below even the $89,095.92 ordinary-bracket tax on $425,574.75 of ordinary taxable income, so it understated the bracket tax and dropped part of the $11,688 qualified-dividend tax and the $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as income and denied the $2,164 traditional IRA deduction, partly offset by excluding the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit. Taxable income came out at $505,148 versus $503,494.75, overstating tax by $529.02 at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no numeric derivation. The correct path applies the $32,200 standard deduction plus $1,070 non-itemizer charity to AGI of $536,764.50, giving $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT. Its $104,535 is $1,971 short, which equals understating taxable income by about $6,160 at 32%; that fits excluding the full $46,308 of 401(k) deferrals instead of $41,650, plus other overstated deductions."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state refund as taxable and itemized, but PolicyEngine excludes the refund, and the $32,200 standard deduction plus $1,070 non-itemizer charity beats the phased-down ~$30.9k SALT itemization (the 0.5% AGI floor wipes out the itemized charity). Its $107,138 is $632 high, about $1,975 of excess taxable income at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $107,377 matches taxable income of about $506,218: AGI of $538,418 less only the bare $32,200 standard deduction. It counted the $4,148 refund, denied the $2,164 IRA deduction, excluded the full $46,308 of 401(k) deferrals instead of $41,650, and left out the $1,070 OBBBA non-itemizer cash charitable deduction, overstating taxable income by $2,723."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages (AGI $525,049) and kept the full $46,308 401(k) exclusion; PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals, giving AGI of $536,764.50. It then itemized $36,959.30 instead of the $33,270 standard-plus-non-itemizer deduction, understating taxable income by $15,405 and tax by $4,929.63."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state refund as taxable, which PolicyEngine excludes, and took the bare standard deduction without the $1,070 OBBBA non-itemizer cash charitable deduction, overstating taxable income. Its $107,077 is $571 above the correct $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049; PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals (AGI $536,764.50). It also itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus $1,070 non-itemizer charity, understating tax by about $4,100."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049 instead of $536,764.50 (only the $41,650 of pro-rated 401(k) deferrals reduce wages), and itemized $34,385.30. Taxable income came out $12,831 too low, cutting tax by $4,105.95 at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Cited a qualified business income deduction even though the household has no business income, and gave no numeric derivation. Its $112,450 overshoots by $5,944 the correct total: $89,095.92 ordinary tax + $11,688 15% qualified-dividend tax + $5,721.98 NIIT on $503,494.75 of standard-deduction taxable income."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: personal exemptions, Pease, 2%-floor miscellaneous deductions, and a pre-TCJA AMT exemption that produced $125,199 of tentative AMT. OBBBA permanently repealed those rules and set the 2026 AMT exemption at $140,200 with a $1,000,000 phase-out threshold, so no AMT applies and tax is $100,783.92 regular tax plus $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (Pease, miscellaneous itemized deductions, an AMT driven by SALT add-backs) and added $25,486 of AMT. Under OBBBA the TCJA rules are permanent, so the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity and owes no AMT, since the $140,200 exemption phases out only above $1,000,000."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of 2026's $32,200, left out the $1,070 OBBBA non-itemizer cash charitable deduction, counted the $4,148 refund, denied the $2,164 IRA deduction, and excluded the full $46,308 of 401(k) deferrals instead of $41,650. That left taxable income at $506,918 versus $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Computed $106,958 of ordinary tax on $423,928, about $18,400 more than the 2026 OBBBA brackets give ($88,569), which points to a pre-TCJA rate schedule. It also subtracted $13,369 of employee ESI premiums from wages and itemized only $23,201 (including sunset-era 2%-floor miscellaneous deductions) instead of taking the larger $32,200 standard deduction plus $1,070 non-itemizer charity."
+us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: uncapped ~$64k of SALT, 2%-floor miscellaneous deductions, Pease, and an $18,221 AMT add-on. Under OBBBA SALT is capped at a phased-down ~$30.9k, miscellaneous deductions stay suspended, Pease is repealed, and the $140,200 AMT exemption leaves no AMT. It also subtracted ESI premiums from wages (AGI $525,049 vs. $536,764.50)."
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for this output, so nothing could be parsed. The correct computation is $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT = $106,505.90."
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 refund, denied the $2,164 IRA deduction, excluded the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. Taxable income came out at $506,218 versus $503,494.75, overstating tax by $871.42."
+us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that itemized deductions, including mortgage interest on the $2.85M balance and unreimbursed employee expenses, wipe out taxable income. In fact no mortgage interest paid was listed, unreimbursed employee expenses are permanently non-deductible under OBBBA, and the SALT cap limits itemizing to ~$30.9k, so taxable income is $503,494.75 and tax is $106,505.90, not $0."
+us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Reached taxable income of $506,218 instead of $503,494.75 by counting the $4,148 refund, denying the IRA deduction, using the unprorated $46,308 401(k) exclusion, and leaving out the $1,070 non-itemizer deduction. It then put NII at $142,667 instead of $150,578.34 (interest + dividends − $3,000 capital loss), giving NIIT of $5,421 rather than $5,721.98."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the entire $10,911 net capital loss in AGI instead of the $3,000 limit, invented $43,118 of mortgage interest from a balance with no stated interest, and netted the capital loss against qualified dividends (a $67,009 preferential base instead of $77,920). It also used wrong bracket breakpoints (24% ending at $247,775) and left out the $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Applied a single-filer-sized $15,700 standard deduction instead of the $32,200 MFJ amount plus $1,070 non-itemizer charity, and understated gross income at $528,543 (correct AGI is $536,764.50). Its $89,067 covers roughly only regular tax and leaves out the $5,721.98 NIIT."
us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It used a $183,600 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It then submitted $33,296.55, which matches neither its own sum of $35,558.20 nor any correct component total."
-us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The federal part was right ($22,599 SS + $6,235 Medicare + $1,620 Additional Medicare = $30,454), but it left out California's mandatory employee SDI. SDI is 1.3% of all $430,000 of wages, or $5,590."
-us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It used the 2024 Social Security wage base of $168,600, even capping the head's $180,000. It applied SDI at 1% with the old $153,164 cap instead of 1.3% uncapped. It also wrongly added the employer ETT and roughly $10,087 of state income tax to the payroll tax total."
-us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It capped the spouse's Social Security wages at $181,800 instead of the 2026 base of $184,500, which understated SS by $167. It applied CA SDI at 1.2% ($5,160) instead of 1.3% on $430,000 ($5,590)."
-us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly derived $30,454 of federal employee payroll tax, then dropped CA SDI (1.3% × $430,000 = $5,590) as 'uncertain'. It submitted $28,586.60, which is below even its own federal figure."
-us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Its federal components were correct ($22,599 + $6,235 + $1,620 = $30,454). It claimed CA SDI was included but added nothing for it, missing the $5,590 SDI at 1.3% of $430,000, and its stated total of $30,437 even misadds the federal parts."
-us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 for both spouses, capping the head's $180,000 as well, instead of the 2026 base of $184,500 ($22,599 total SS). It also applied CA SDI at 1.2% instead of the 2026 rate of 1.3%."
-us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It capped both spouses at a $176,100 Social Security base instead of $184,500, and used a 1.1% SDI rate instead of 1.3% on $430,000. It then submitted $27,306, which does not match its own $34,421.40 computation."
-us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It computed Social Security for only one capped wage base ($11,253), even though each spouse owes 6.2% up to their own $184,500 cap ($11,160 + $11,439 = $22,599). It also left out California SDI of $5,590 entirely."
-us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its $22,407 of Social Security implies a spouse wage base of about $181,400 instead of $184,500. Its $4,080 of SDI reflects a per-worker wage cap, when 2026 CA SDI is 1.3% of all $430,000 with no ceiling ($5,590)."
-us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% on a $153,164 per-worker cap ($3,675.94), but the wage ceiling ended in 2024 and the 2026 rate is 1.3% on all $430,000 ($5,590). It also used a $181,800 Social Security base instead of $184,500."
-us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a $184,200 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)."
-us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its $21,242 of Social Security comes from capping both spouses at an old wage base of about $171,300, including the head's $180,000, instead of taxing the head's full $180,000 plus the spouse's $184,500 ($22,599)."
-us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out California's mandatory employee SDI ($5,590 at 1.3% of $430,000). Its federal figure of $31,398 also misstates the correct $30,454 federal total."
-us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted employer-sponsored insurance premiums from the FICA and SDI wage bases, although the gross wages of $180,000 and $250,000 are the payroll tax base. It also applied SDI at about 1.1% instead of 1.3% of $430,000."
-us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% ($4,730) instead of the 2026 rate of 1.3% ($5,590). Its federal portion is also understated because its Social Security wage base was too low, when it should tax the head's full $180,000 plus the spouse's $184,500."
-us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare, left out California SDI ($5,590), and understated the federal portion. The correct federal total is $30,454 once the $1,620 Additional Medicare Tax and the $184,500 per-worker SS base are included."
-us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced the head's FICA wages to $173,111 and the spouse's Medicare/SDI wages by their employer insurance premiums, although payroll tax applies to the stated gross wages. It also used a $181,200 SS base instead of $184,500 and a 1.1% SDI rate instead of 1.3%."
-us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $33,814 is $2,230 short. That is consistent with SDI at the outdated 1.1% rate ($4,730 instead of $5,590 at 1.3%) combined with a Social Security base below the 2026 base of $184,500."
-us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It capped both spouses at a $176,100 Social Security base ($10,918.20 each), including the head's $180,000, instead of using the 2026 base of $184,500. It also left out California SDI of $5,590 entirely."
-us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% with a $165,384 per-worker cap ($3,969.22), but 2026 SDI is 1.3% with no wage ceiling ($5,590). It also used a $182,100 Social Security base instead of $184,500."
-us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It stated that California has no mandatory employee payroll tax and so left out CA SDI. SDI is withheld at 1.3% of all $430,000 of wages ($5,590) on top of the correct $30,454 federal total."
-us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It explicitly skipped the 0.9% Additional Medicare Tax on combined wages above the $250,000 MFJ threshold ($1,620) and left out CA SDI ($5,590). It also understated employee Social Security and Medicare below the correct $22,599 and $6,235."
-us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It capped Medicare at the Social Security wage base, although Medicare's 1.45% applies to all $430,000 of wages ($6,235). Its $13,905 total also falls far short of the $22,599 in SS, the $1,620 Additional Medicare Tax, and the $5,590 CA SDI."
-us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It used a $183,600 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)."
-us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It had the SDI rate right at 1.3% ($5,590), but its total is $623 too high. That leaves federal tax of $31,077 instead of $30,454, which fits a spouse Social Security base about $10,000 above the 2026 base of $184,500."
-us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The federal part was correct at $30,454, but its $35,614 total reflects CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
-us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The federal part was correct at $30,454, but its $35,614 total reflects CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
-us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the employer insurance premiums ($6,889 and $6,480) as pretax reductions to FICA and SDI wages, shrinking the base to $416,631. The payroll tax base is the full stated gross wage of $430,000, which gives SS $22,599, Medicare $6,235, Additional Medicare $1,620, and SDI $5,590."
-us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its federal components were correct ($22,599 + $6,235 + $1,620), but it computed CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
-us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the employer insurance premiums as pretax and subtracted them from the FICA, Additional Medicare, and SDI wage bases, which understated every component. Payroll tax applies to the full $430,000 of gross wages, giving $36,044 with SDI at 1.3%."
-us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only OASDI, HI, and Additional Medicare Tax, so it left out CA SDI ($5,590 at 1.3% of $430,000). Yet its $34,515 total is well above the correct federal amount of $30,454, so it also misstated the federal components."
-us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% with a per-worker cap ($4,226), but 2026 SDI is 1.3% of all wages with no ceiling ($5,590). It also capped the spouse at about $182,000 for Social Security instead of $184,500."
-us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% up to an 'SDI wage limit', but SB 951 removed the SDI wage ceiling. The 2026 rate is 1.3% on all $430,000 ($5,590)."
-us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted $13,369 of health premiums from FICA wages, although payroll tax applies to the stated gross $430,000. It also applied SDI at 1.1% on an invented $83,067 per-person cap ($1,827) instead of 1.3% uncapped ($5,590)."
-us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It reduced FICA wages by the employer insurance premiums, used the obsolete $168,600 Social Security cap instead of $184,500, and applied SDI to capped wages. The correct SDI is 1.3% on all $430,000."
-us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no reasoning, so there was no answer to score."
-us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its federal components were correct ($11,160 + $11,439 + $6,235 + $1,620), but it applied uncapped CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)."
-us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It claimed employee Social Security and Medicare were 'offset by deductions/exemptions' and answered $0, but income-tax deductions do not reduce FICA or SDI. The household owes $22,599 SS + $6,235 Medicare + $1,620 Additional Medicare + $5,590 CA SDI."
-us,scenario_005,payroll_tax,ox-alpha,llm_error,household_unit_or_filing_status,False,"It computed Additional Medicare Tax on the spouse's wages above the $200,000 single-filer withholding threshold ($2,250). For joint filers it applies to combined wages above $250,000 ($1,620). It also wrongly stated that California has no mandatory employee payroll tax and left out SDI of $5,590."
-us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It applied SDI at 1.1% on a $153,164 per-worker cap ($3,369.60), but the cap was removed in 2024 and the 2026 rate is 1.3% of all $430,000 ($5,590). It also capped both spouses at the 2025 Social Security base of $176,100 instead of $184,500."
-us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It counted only federal Social Security, Medicare, and Additional Medicare and left out California SDI ($5,590 at 1.3% of $430,000). Its federal total of $25,474 is also below the correct $30,454."
+us,scenario_005,payroll_tax,claude-fable-5,llm_error,other,False,"The model's own working added up to $35,558.20, using a $183,600 wage base and CA SDI at 1.2%. It then submitted $33,296.55, a number its calculation doesn't support. Even its working total was low because the 2026 CA SDI rate is 1.3% ($5,590), not 1.2% ($5,160), and the Social Security wage base is $184,500."
+us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It got all the FICA parts right ($22,599 + $6,235 + $1,620 = $30,454) but left out California's mandatory employee SDI. That tax is 1.3% of the full $430,000 with no cap, which adds $5,590."
+us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 Social Security wage base of $168,600 for both earners, which also wrongly capped the head's $180,000. It applied SDI at 1% capped at $153,164, though SDI has been uncapped since 2024 and is 1.3% in 2026. It also added a made-up ETT amount and $10,086.52 of state income tax, none of which is employee payroll tax."
+us,scenario_005,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It never calculated anything and simply entered 0. The correct total is $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare Tax + $5,590 CA SDI on $430,000 of wages."
+us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also capped the spouse's Social Security wages at $181,800 instead of $184,500, which understated that tax by $167."
+us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It calculated FICA correctly at $30,454, then dropped CA SDI as uncertain. The correct SDI is 1.3% × $430,000 = $5,590. It then submitted $28,586.60, which is even below its own FICA total."
+us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It mentioned CA SDI but wrongly said it was already included, then submitted roughly the FICA total alone ($30,437 vs. its own $30,454). That left out the $5,590 of SDI at 1.3% of all $430,000 in wages."
+us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That wrongly capped the head's $180,000 as well as the spouse's wages. It also used 1.2% for CA SDI instead of the 2026 rate of 1.3%."
+us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its working reached $34,421.40 using a $176,100 wage base and SDI at 1.1%. It then submitted $27,306, which matches nothing it calculated. The correct figures are a $184,500 cap on the spouse only and SDI at 1.3% uncapped, which gives $5,590."
+us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It charged Social Security once for the whole household ($11,253). Each spouse owes 6.2% on their own wages up to the cap: $11,160 for the head plus $11,439 for the spouse. It also left out CA SDI of $5,590."
+us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its CA SDI of $4,080 is far below the correct $5,590, which is 1.3% of all $430,000 with no wage cap since 2024. Its Social Security figure of $22,407 also reflects a spouse wage base below the 2026 $184,500."
+us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It capped CA SDI at $153,164 per worker at 1.2%, totaling $3,675.94. SB 951 removed the SDI wage ceiling starting in 2024, and the 2026 rate is 1.3%, so SDI is $5,590. It also used a $181,800 Social Security base instead of $184,500."
+us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of 1.3% ($5,590). It also used a $184,200 Social Security wage base instead of $184,500."
+us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its Social Security figure of $21,242 implies it capped both earners near $171,000. The head's $180,000 falls below the 2026 base of $184,500 and is fully taxable."
+us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI on $430,000, worth $5,590."
+us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted the ESI premiums from FICA and SDI wages, but the engine taxes the full $180,000 and $250,000 of gross wages. Its SDI also falls short of 1.3% of the full $430,000."
+us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% ($4,730) instead of the 2026 rate of 1.3% ($5,590). Its total implies Social Security of about $21,115, which is below the correct $22,599 from the $184,500 cap applied to the spouse only."
+us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out both the $1,620 Additional Medicare Tax and the $5,590 CA SDI. Its $26,588 is also below the correct FICA total of $30,454."
+us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It cut the wage base to $173,111 and $243,520 by subtracting ESI premiums, but the engine applies payroll taxes to the full gross wages. It also used 1.1% for CA SDI instead of 1.3% of $430,000 ($5,590)."
+us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $33,814 is $2,230 short. The correct figures are $22,599 + $6,235 + $1,620 + $5,590. That shortfall means its CA SDI was well below 1.3% of the full uncapped $430,000."
+us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It left out California's mandatory 1.3% employee SDI ($5,590). It also capped both earners at the old $176,100 Social Security base, though the head's $180,000 falls below the 2026 base of $184,500."
+us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It capped CA SDI at $165,384 per worker at 1.2% ($3,969.22). In 2026 SDI has no wage ceiling and the rate is 1.3%, so SDI is $5,590. It also used a $182,100 Social Security base instead of $184,500."
+us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly said California has no mandatory employee state payroll tax. CA SDI is required at 1.3% of all wages in 2026, which adds $5,590 to its correct $30,454 of FICA."
+us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applies, but combined wages exceed the $250,000 MFJ threshold by $180,000, which adds $1,620. It also left out the $5,590 CA SDI and understated Social Security and Medicare below the correct $28,834."
+us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $13,905 is less than half of the correct federal FICA ($30,454) alone. It treated the Social Security cap as limiting all FICA, even though Medicare's 1.45% applies to all $430,000. It also left out the $5,590 CA SDI."
+us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also used a $183,600 Social Security wage base instead of $184,500."
+us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the right 1.3% SDI rate, but its $36,667 is $623 above the sum of $22,599 + $6,235 + $1,620 + $5,590. That means it overstated Social Security, either by applying a wage base above $184,500 to the spouse or by miscalculating."
+us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $35,614 equals correct FICA ($30,454) plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3%, which gives $5,590."
+us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $35,614 equals correct FICA ($30,454) plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3%, which gives $5,590."
+us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $13,369 of ESI premiums as reducing wages, then applied Social Security, Medicare, Additional Medicare and 1.3% SDI to $416,631. The engine applies these taxes to the full $430,000 of gross wages."
+us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It calculated CA SDI as $5,160 (1.2%) instead of 1.3% × $430,000 = $5,590. All other components ($22,599, $6,235, $1,620) were correct."
+us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the ESI premiums as reducing wages, which shrank the base for FICA, Additional Medicare and 1.3% SDI to $416,631. The engine taxes the full $180,000 and $250,000 of gross wages."
+us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only OASDI, HI and Additional Medicare Tax and never mentions CA SDI. Its $34,515 is $1,529 short of the correct total, so it did not apply SDI as 1.3% of the full $430,000 ($5,590)."
+us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It capped CA SDI and used 1.2%, getting $4,226. SDI has had no wage ceiling since 2024 and is 1.3% in 2026, which gives $5,590. It also used a spouse Social Security base of about $182,000 instead of $184,500."
+us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% only up to an SDI wage limit. That limit was repealed in 2024, and the 2026 rate is 1.3% of all $430,000 ($5,590)."
+us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted $13,369 of ESI premiums to get $416,631 of FICA wages, though the engine taxes the full $430,000. It also applied SDI at 1.1% to an invented $83,067 cap per person instead of 1.3% uncapped ($5,590)."
+us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It reduced wages by the ESI premiums and capped Social Security at the 2024 base of $168,600 instead of $184,500. It also applied CA SDI to capped wages, though SDI is 1.3% of all $430,000."
+us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score."
+us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It got Social Security ($22,599), Medicare ($6,235) and Additional Medicare ($1,620) right, but used 1.2% for CA SDI ($5,160). The 2026 rate is 1.3%, which gives $5,590."
+us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It claimed payroll taxes are offset by income-tax deductions and exemptions, but employee FICA and SDI are not reduced by income-tax deductions. Wages of $430,000 produce $22,599 Social Security, $6,235 Medicare, $1,620 Additional Medicare and $5,590 CA SDI."
+us,scenario_005,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It said no CA employee payroll tax is modeled and so left out the 1.3% SDI ($5,590). It also used the $200,000 single-filer threshold on the spouse's wages for Additional Medicare Tax ($2,250). The correct rule applies the $250,000 MFJ threshold to combined wages, giving $1,620."
+us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% up to a $153,164 cap per earner, even though the cap was repealed in 2024 and the 2026 rate is 1.3% of all $430,000. It also used the 2025 Social Security base of $176,100, which wrongly capped the head's $180,000."
+us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It counted only federal FICA and left out the $5,590 CA SDI. Its $25,474, which it said included Additional Medicare Tax, is also below the correct FICA total of $30,454."
us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $22,800 of mortgage interest on the first $1M of the $2.85M balance, but no interest is listed and unlisted amounts are $0. It also counted charity and employee expenses, which cut taxable income to $490,509 instead of $521,425.72. Its own bracket math on that base came to about $38,832, and the $45,856 it submitted is an unexplained upward 'refinement'; it also set the $156.93 exemption credits to zero."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 traditional 401(k) request even though each spouse's combined traditional+Roth request of $27,240 exceeds the $24,500 limit, so its CA AGI of $532,106 is $4,658.50 below $536,764.50. It also itemized about $21,560 including charity and employee expenses, instead of the $17,280 real-estate-tax base limited to $15,338.79. That left taxable income about $10,880 short of $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a California deduction for federal income tax paid that does not exist, plus employer health premiums, and used an invented 12% rate above $325,000. It then took off about $21,621 of made-up 'standard deduction credit' and other nonrefundable credits. The only nonrefundable credit here is the phased-down MFJ exemption credit, subtracted from the tax on $521,425.72 of taxable income: $156.93 from $41,208.45 on the frozen reference's projected 2026 amounts, or $150 from $41,369.87 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $60,000 of mortgage interest on the $1M-limited share of the $2.85M balance although no interest is listed. That raised itemized deductions to $79,854 instead of $15,338.79 and cut taxable income to $453,116 instead of $521,425.72. It lands near the target only because its bracket math overstated tax on that base (~$38,800 vs ~$34,856 on the frozen reference's projected 2026 schedule and ~$35,017 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), and it used $560 of exemption credits instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to CA income and imputed about $35,000 of unlisted mortgage interest, reaching about $595,000 of taxable income instead of $521,425.72. It then reported $36,767, far below what the California schedule gives on its own $595,000 base (about $48,050 on the frozen reference's projected 2026 schedule, and slightly more on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026). Both its income base and its use of the rate schedule are wrong."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put taxable income at about $560,000 by partly including the $70,000 tax-exempt pension and imputing mortgage interest; the correct base is $521,425.72. It also applied rates up to 11.3%, but the 10.3% MFJ bracket starts above roughly $740,000. At this income every marginal dollar is taxed at 9.3%, giving $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 of traditional 401(k) deferrals although each spouse's $27,240 combined traditional+Roth request exceeds the $24,500 limit, so its CA AGI of $532,106 is $4,658.50 short of $536,764.50. It also itemized $22,454 including charity and employee expenses, instead of the $15,338.79 left from the $17,280 real-estate-tax base after the 6% limitation. Its taxable income of $509,652 is $11,774 below $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It set CA AGI at $534,270, $2,494.50 below $536,764.50, by excluding the full $46,308 traditional 401(k) request despite the $24,500 per-person combined-deferral cap, net of the $2,164 IRA deduction. It itemized $23,017 including charity and employee expenses and never applied the 6% limitation, though AGI exceeds the $504,411 threshold; the correct figure is $15,338.79. It also used an unphased $500 exemption credit instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $50,000 of mortgage interest (5% on $1M of the $2.85M balance) although no interest is listed and unlisted amounts are $0. That cut taxable income to $464,416 instead of $521,425.72. Its bracket math then understated tax even on that base (~$35,900 on the frozen reference's projected 2026 schedule and ~$36,068 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026, not $32,000), and it ignored the $156.93 exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It used about $534k of CA AGI by excluding the full $46,308 401(k) request despite the $24,500 combined-deferral cap. It itemized about $19.9k of property tax plus charity with no 6% high-income limitation instead of $15,338.79, leaving taxable income around $514k rather than $521,425.72. It also treated the exemption credits as fully phased out, when some remain after the $6-per-$2,500 reduction: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $518,564 is within about $2,900 of $521,425.72; it started from federal AGI of $538,418 without removing the $4,148 refund and deducted $19,854 of itemized deductions with no limitation. But its bracket math produced $42,151 where the frozen reference's projected 2026 MFJ schedule gives about $40,942 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,104. It also left out the $156.93 of phased-down exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It taxed only $497,700, which is $23,726 below $521,425.72. That figure matches CA AGI of $520,901, built by treating the $13,369 of employer insurance premiums as a wage exclusion, minus about $23,200 of itemized deductions with charity and employee expenses and no limitation. It also used $300 of exemption credits instead of the phased-down $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $512,889 is $8,537 below $521,425.72. That gap fits excluding the full traditional 401(k) request despite the $24,500 combined-deferral cap and deducting more than the $15,338.79 of limited real-estate-tax itemized deductions. It also said no nonrefundable credits apply at this AGI, but MFJ exemption credits survive the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short of $536,764.50, because it excluded the full $46,308 traditional 401(k) request although each spouse's $27,240 combined request exceeds the $24,500 limit (net of the $2,164 IRA deduction). Its $18,663 of itemized deductions after limitation adds charity to the $17,280 real-estate-tax base; the correct figure is $15,338.79. Its $310 exemption credit ignores the $6-per-$2,500 phase-down, which leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference)."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It deducted mortgage interest on the $2.85M balance, for $73,017 of itemized deductions, although no mortgage interest is listed and unlisted amounts are $0. The itemized base is the $17,280 of real estate taxes limited to $15,338.79. Its mistake cut taxable income to $461,253 instead of $521,425.72, and it used a $288 exemption credit instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $31,215 matches about $415,700 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $413,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $106,000 to $108,000 below $521,425.72. That fits imputing a large mortgage-interest deduction on the $2.85M balance or dropping investment income. The frozen reference taxes $521,425.72 at up to 9.3% for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $501,848 of taxable income equals federal AGI minus $13,369 of employer insurance premiums (treated as a wage exclusion) minus $23,201 of itemized deductions with charity, employee expenses and no limitation. Employer premiums are not subtracted, and the itemized base is $17,280 of real estate taxes limited to $15,338.79, giving $521,425.72. That left its tax about $856 short."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer insurance premiums as if they were pre-tax wage reductions and excludes the full $46,308 401(k) request; the correct CA AGI is $536,764.50. It also itemized $19,854 (property tax plus charity) with no 6% high-income limitation instead of $15,338.79. Its taxable income ended about $20,400 too low."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $33,816 matches about $443,600 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $441,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $78,000 to $80,000 below $521,425.72. That fits over-deducting, such as mortgage interest on the $2.85M balance. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI without removing the $4,148 state refund and itemized property tax, charity and employee expenses with no 6% limitation, giving about $515,500 of taxable income. The frozen reference's projected 2026 MFJ schedule taxes that at about $40,650 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 at about $40,820, so its $42,150 overshoots by roughly $1,300 to $1,500 through misapplied bracket thresholds. It also subtracted none of the $156.93 exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its reasoning mentions 'pre-tax deductions' plus standard/itemized deductions, and $36,918 matches about $477,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $475,200 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $44,400 to $46,300 below $521,425.72. That fits excluding employer health premiums and claiming extra itemized deductions. The correct base is $536,764.50 of CA AGI minus $15,338.79 of limited real-estate-tax itemized deductions."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270 (the full $46,308 401(k) exclusion, missing the $24,500 combined-deferral cap) and itemized real estate taxes plus charity and employee expenses, reaching about $515,000 of taxable income instead of $521,425.72. It then reported a round $40,000 rather than the ~$40,450 its own base gives after the $156.93 exemption credits on the frozen reference's projected amounts (about $40,620 after $150 of credits on the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026)."
-us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI and deducted the full $10,911 net capital loss instead of the $3,000 limit. It then invented a $2,000 CA capital-loss cap, producing CA AGI of $449,439 instead of $536,764.50. It also took an ~$11,000 standard deduction where $15,338.79 of limited itemized real estate taxes applies."
-us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $522,970 is within roughly $1,550 of $521,425.72, but its bracket math gave $41,942 where the frozen reference's projected 2026 MFJ schedule yields about $41,352 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,513. It also claimed no nonrefundable credits, when phased-down exemption credits apply: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference. It took the standard deduction even though the $15,338.79 of limited itemized real estate taxes exceeds the $11,412 standard deduction."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It offered only a generic statement, and $20,677 matches about $302,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $300,500 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026). That drops roughly $219,000 from the correct $521,425.72 base, which is wages net of capped 401(k) deferrals, $65,936 of interest and $87,642 of dividends taxed as ordinary income, minus the $3,000 capital loss, the $4,148 refund and $15,338.79 of itemized deductions."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no concrete derivation, and $37,099 matches about $479,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $477,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $42,500 to $44,300 below $521,425.72. That fits over-deducting, such as imputed mortgage interest or excluded employer health premiums. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It cut wages by the $13,369 of employer insurance premiums as if they were pre-tax, on top of the full $46,308 traditional 401(k) request, and itemized beyond the $17,280 real-estate-tax base. Its result matches about $497,600 of taxable income instead of $521,425.72. That left its tax about $2,218 short."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $519,427 is $1,999 below $521,425.72, so its combination of CA AGI and itemized deductions understated the base (correct: $536,764.50 minus $15,338.79). The resulting ~$186 of missing 9.3% tax, partly offset by its bracket and credit estimates, left it $27.51 short of $41,051.51."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its answer is $98.49 too high, which equals about $1,059 of excess taxable income at 9.3%. Its 'California additions' step adds income that is not in CA AGI. CA AGI is federal AGI minus the $4,148 refund, $536,764.50, with no add-back of the $8,961 tax-exempt interest or the pension."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It added the $8,961 of tax-exempt interest to CA income, but that interest is not part of the $536,764.50 CA AGI. Its full $46,308 401(k) exclusion and its itemized deductions with charity and employee expenses more than offset that addition. Taxable income ended about $9,000 below $521,425.72 and the tax $885.51 short."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer insurance premiums from wages and excluded the full $46,308 401(k) request, giving CA AGI of $520,901 instead of $536,764.50. It itemized $22,294.58 (charity plus employee expenses above 2%, limited), which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading, where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing the $15,338.79 real-estate-tax-only figure; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference. It also used the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 exemption credit is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It kept the $4,148 state tax refund in CA income, although California subtracts it. It also took the standard deduction instead of itemizing the $17,280 of real estate taxes, which are limited to $15,338.79 and still exceed the $11,412 standard deduction. Both errors pushed taxable income above $521,425.72 and left the answer $738.49 too high."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $492,027, which is $29,399 below $521,425.72, because its estimated deductions and exclusions exceed the actual $15,338.79 of limited real-estate-tax itemized deductions taken against CA AGI of $536,764.50. Its bracket and exemption-credit math on that base is consistent with the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($38,635.79 of tax on $492,027 before credits) as well as with the frozen reference's projected schedule, so the whole miss comes from the understated income base."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its CA AGI of $520,901 comes from subtracting the $13,369 of employer insurance premiums from wages and excluding the full $46,308 401(k) request; the correct AGI is $536,764.50. It itemized $22,294.58 including charity and employee expenses, which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading ($17,280 of real estate taxes, $2,574 of charity and $3,429.98 of employee expenses above 2% of AGI, less the $989.40 high-income limitation, 6% of AGI above $504,411), where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing $15,338.79; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference. It also held the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 of credits is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $48,720 matches about $604,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $602,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $80,600 to $82,500 above $521,425.72. That fits adding the $70,000 tax-exempt private pension and the $8,961 tax-exempt interest to CA income. Neither is part of the $536,764.50 CA AGI."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short, by excluding the full $46,308 401(k) request despite the $24,500 combined-deferral cap. It kept charity and 2%-floor employee expenses in the itemized base before the 6% cutback, reaching about $512,000 of taxable income instead of $521,425.72. It also used about $280 of exemption credits instead of what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It itemized $22,934 (property tax, charity, and employee expenses over the 2% floor) with no 6% high-income limitation. The correct figure is $15,338.79: the $17,280 real-estate-tax base cut by 6% of AGI over $504,411. Taxable income therefore fell to $515,484 instead of $521,425.72, even though its $538,418 AGI wrongly kept the $4,148 state refund that California subtracts."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 of traditional 401(k) deferrals, giving CA AGI of $534,270 instead of $536,764.50, although each spouse's $27,240 combined request exceeds the $24,500 limit. It added $2,574 of charity to the $17,280 real-estate-tax base, for $18,100 of itemized deductions instead of $15,338.79, leaving taxable income of $516,170, $5,256 low. It also claimed $314 of exemption credits where the $6-per-$2,500 phase-down leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference)."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer insurance premiums from federal AGI, giving CA AGI of $525,049 instead of $536,764.50. It itemized $23,201 with charity and employee expenses and no 6% limitation instead of $15,338.79, so taxable income was $501,848 rather than $521,425.72. It also applied no exemption credits instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and $8,961 of tax-exempt interest to CA AGI and subtracted employer health premiums, reaching $604,010 instead of $536,764.50. Neither tax-exempt item belongs in CA AGI. With about $21,622 of itemized deductions and no high-income limitation, its $582,388 of taxable income is about $61,000 too high."
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so there was nothing to score. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts, giving $41,051.51; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40."
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short, because it excluded the full $46,308 traditional 401(k) request despite the $24,500 combined-deferral cap. It itemized $21,837 after limitation with charity and employee expenses included instead of $15,338.79, giving taxable income of $512,433 rather than $521,425.72. Its $258 exemption credit is also larger than what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference."
-us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap, which California does not apply to property tax, imputed unlisted mortgage interest, and concluded that deductions wipe out all tax. With CA AGI of $536,764.50 and only $15,338.79 of itemized deductions, taxable income is $521,425.72. Tax is $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026."
-us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI of $538,418 without subtracting the $4,148 state tax refund that California excludes. It also over-applied the high-income limitation, cutting its $19,854 itemized total to $11,912; the actual 6% reduction on AGI over $504,411 is only $1,941.21 on the $17,280 real-estate-tax base, giving $15,338.79. Its taxable income of about $526,500 is therefore about $5,074 too high."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It skipped the traditional 401(k) exclusion, deducted the full $10,911 net capital loss instead of the $3,000 limit, and kept the $4,148 refund. It then claimed about $33,544 of tax on $565,375 of taxable income, where the frozen reference's projected 2026 MFJ schedule gives about $45,300 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $45,457. It also subtracted an inflated $892 of exemption credits instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction instead of the $15,338.79 of limited itemized real estate taxes and reached $495,603 of taxable income. It then computed only $29,423 of tax on that amount, while the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $38,968: $6,403.94 through the 8% bracket plus 9.3% above $145,448 (the frozen reference's projected schedule gives about $38,800: about $6,549 through the 8% bracket plus 9.3% above roughly $148,700). The rate-schedule error is the bigger of the two."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It invented about $22,800 of mortgage-interest deduction from the loan balance, even though no interest paid was listed, so that amount should be $0. It also deducted the full $46,308 of 401(k) deferrals despite the $24,500 combined traditional+Roth limit. Its stated tax of about $38,832 was then replaced with an unexplained $45,856 that does not follow from its own working."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its California AGI of $532,106 deducts the full $46,308 of traditional 401(k) deferrals and also the $2,164 of IRA contributions. The 401(k) exclusion is capped at about $20,825 per spouse under the $24,500 combined limit, and the IRA contributions are not deductible for covered workers at this income. The correct California AGI is $538,928, and the surviving exemption credit is $114, not about $210."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a made-up deduction for federal income tax paid, which California does not allow, and also subtracted employer-paid insurance premiums. It applied a 12% bracket that does not apply at $518k, and it invented about $21,621 of nonrefundable credits when only a $114 exemption credit applies."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed the California tax and entered $0 as a placeholder. California taxable income is $518,408, which produces $40,975 of tax after the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $60,000 of mortgage interest from the $2.85M balance even though no interest paid was given, so that amount should be $0. It left out the miscellaneous employee-expense deduction and deducted the full 401(k) deferrals despite the $24,500 combined limit. It also claimed a $560 exemption credit that phases down to $114 at this AGI."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to California income, but it is excluded. It also imputed $35,000 of mortgage interest when none was listed. The final $36,767 does not follow from its own taxable income of about $595,000."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put part of the tax-exempt pension into California income and arrived at about $560,000 of taxable income instead of $518,408. It then applied rates up to 11.3%, but the top slice of this income is taxed at 9.3%."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its California AGI of $532,106 subtracts the full $46,308 of 401(k) deferrals and the non-deductible $2,164 of IRA contributions. The 401(k) exclusion is capped at about $20,825 per spouse because traditional plus Roth desires exceed the $24,500 limit. As a result, taxable income came out about $8,756 too low, and it used a $256 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It excluded the full $23,154 traditional 401(k) per spouse and ignored the $24,500 combined traditional+Roth cap, which understates California AGI by $4,658. It never applied the 6% high-income itemized-deduction limitation, which cuts $2,320. It also subtracted an unphased $500 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It assumed $50,000 of deductible mortgage interest even though no interest paid was given, so that amount should be $0. It left out the miscellaneous employee-expense deduction. Its bracket arithmetic also gave about $32,000, which is too low even for its own $464,416 of taxable income."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It started from California AGI of about $534k and deducted the full 401(k) deferrals despite the $24,500 combined limit. It left out the $2,986 deduction for employee business expenses above the 2% floor and skipped the 6% itemized limitation. It also treated the exemption credits as fully phased out, when $114 survives."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income of $518,564 is nearly correct, but its bracket computation gave $42,151 instead of the roughly $41,100 that the 2026 MFJ schedule produces. It also said no nonrefundable credits apply and dropped the $114 exemption credit that survives the phase-out."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $497,700 is $20,708 below the correct $518,408, which is California AGI of $538,928 minus $20,520 of limited itemized deductions. It also subtracted a $300 exemption credit that phases down to $114 at this AGI."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its taxable income of $512,889 is $5,519 too low. That gap comes mostly from excluding the full $46,308 of traditional 401(k) deferrals instead of the roughly $41,650 allowed under the $24,500 combined traditional+Roth limit. It also wrongly said no nonrefundable credits apply, when a $114 exemption credit does."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which deducts the full 401(k) deferrals despite the $24,500 combined limit and is $4,658 too low. It left out the $2,986 deduction for employee business expenses above the 2% floor, giving $18,663 of itemized deductions instead of $20,520. It also took a $310 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $73,017 of itemized deductions includes about $50,000 of mortgage interest that it imputed from the loan balance, even though no interest paid was listed. Without it, the limited itemized total is $20,520. It also used the $534,270 AGI, which ignores the 401(k) combined-limit cap."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $31,215 is about $9,760 short, which is consistent with understating taxable income by roughly $105,000. The correct calculation is California AGI of $538,928 minus $20,520 of limited itemized deductions, giving $518,408 of taxable income, then the 2026 MFJ schedule and the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $501,848 of California taxable income is $16,560 below the correct $518,408. That figure matches subtracting the $13,369 of employer-paid insurance premiums (not part of reported wages) and taking unlimited itemized deductions. Its bracket math then overstated tax on that smaller base, so the two errors only partly cancel."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 equals $534,270 minus the $13,369 of employer-sponsored insurance premiums. Those premiums are employer-paid and were never in gross wages. It also deducted the full 401(k) deferrals despite the $24,500 combined limit, and it left the $2,986 employee-expense deduction out of its $19,854 itemized total."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $33,816 is about $7,160 short, which implies roughly $77,000 too little taxable income. The correct taxable income is $518,408 (California AGI of $538,928 minus $20,520 of limited itemized deductions), with a $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from federal AGI without the $4,148 California subtraction for the state tax refund. It did not apply the 6% high-income itemized limitation or the $114 exemption credit, which overstated tax by about $1,175."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $36,918 is about $4,057 short, which implies roughly $43,600 too little taxable income, consistent with imputing deductions that were not listed, such as mortgage interest. The correct base is California AGI of $538,928 minus $20,520 of limited itemized deductions."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which deducts the full $46,308 of 401(k) deferrals. Because traditional plus Roth desires exceed the $24,500 per-person limit, only about $41,650 is excludable, and the correct California AGI is $538,928. It then rounded the result to a flat $40,000."
+us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI and deducted the full $10,911 net capital loss instead of the $3,000 limit. It then restricted the California capital loss to $2,000, but the MFJ limit is $3,000. It also used an $11,000 standard deduction even though itemized deductions of $20,520 are larger."
+us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the roughly $11,300 standard deduction instead of itemizing. Property tax, charity and employee expenses above the 2% floor total $20,520 even after the high-income limitation. It also deducted the full 401(k) deferrals despite the $24,500 combined limit and dropped the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its $20,677 is about half the correct amount. California taxable income of $518,408, with the top slice taxed at 9.3%, produces $41,089 of tax before the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $37,099 is $3,876 short, which implies roughly $41,700 too little taxable income compared with the correct $518,408. That figure is California AGI of $538,928 minus $20,520 of limited itemized deductions."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took pretax health premiums out of wages, but the listed employer-sponsored premiums are employer-paid and were never in the reported gross wages. It also excluded the full 401(k) deferrals despite the $24,500 combined traditional+Roth limit. Both errors understate California AGI."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $519,427 is $1,019 above the correct $518,408 (California AGI of $538,928 minus $20,520 of itemized deductions after the 6% limitation). That gap accounts for its $49 overstatement."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $41,150 is higher than the $41,089 tax before any credit, so it implies about $1,900 more taxable income than the correct $518,408. That means it undercounted the $20,520 of limited itemized deductions, which include $2,986 of employee expenses above the 2% floor."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated the $8,961 of federally tax-exempt interest as taxable in California, which PolicyEngine does not do. Even so, it came out $809 low, which implies taxable income about $8,700 below $518,408. That means it over-subtracted elsewhere, including excluding the full 401(k) deferrals despite the $24,500 combined limit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 subtracts the $13,369 of employer-sponsored premiums, which are employer-paid and not in gross wages. It also subtracts the full $46,308 of 401(k) deferrals instead of the roughly $41,650 allowed under the $24,500 combined limit. It used a $222 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It kept the $4,148 state-tax refund in California income instead of subtracting it. It also used the standard deduction even though California itemized deductions of $20,520 after limitation are larger than the $11,412 standard deduction."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $492,027 is $26,381 below the correct $518,408, which means its itemized deductions far exceed the allowable $20,520. It also applied two full personal exemption credits instead of the $114 left after the phase-out."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 subtracts the $13,369 of employer-paid insurance premiums, which were never in gross wages. It also subtracts the full 401(k) deferrals despite the $24,500 combined traditional+Roth limit. It used a $222 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $48,720 is $7,745 too high, which implies about $83,000 of extra taxable income. That is consistent with adding the $70,000 tax-exempt pension and the $8,961 of tax-exempt interest to California income, and neither belongs there."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which excludes the full $46,308 of 401(k) deferrals and ignores the $24,500 combined traditional+Roth cap that limits the exclusion to about $20,825 per spouse. It also took about $280 of exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It took $22,934 of itemized deductions without the 6% high-income limitation, which reduces them by $2,320 to $20,520. It also skipped the $4,148 SALT-refund subtraction, and it used a $177 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $2,986 deduction for employee business expenses above the 2% floor. It used California AGI of $534,270, which excludes the full 401(k) deferrals despite the $24,500 combined limit and is $4,658 too low. It also took a $314 exemption credit instead of the $114 left after the phase-out."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its California AGI of $525,049 is federal AGI of $538,418 minus the $13,369 of employer-paid premiums, which are not in wages. It also skipped the SALT-refund subtraction and did not apply the 6% itemized limitation. It took no exemption credit, when $114 survives the phase-out."
+us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and the $8,961 of tax-exempt interest to California AGI and got about $604,010. Both are excluded, and the correct California AGI is $538,928."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer and no explanation for state_income_tax_before_refundable_credits, so there was no value to score."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $538,418 excludes the full $46,308 of traditional 401(k) deferrals. Each spouse's $27,240 traditional+Roth desire exceeds the $24,500 limit, so only about $20,825 per spouse is excludable. That left California AGI $4,658 too low, and it used a $258 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap to California, which does not have that cap, and it invented mortgage interest that was never listed. It then concluded that taxable income was eliminated, when California taxable income is $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It over-applied the high-income itemized limitation and cut $19,854 down to $11,912, but the 6%-of-excess-AGI reduction is only $2,320. It also left out the $2,986 employee-expense deduction and skipped the $4,148 SALT-refund subtraction."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It deducted the full $10,911 net capital loss instead of $3,000, did not subtract any 401(k) deferrals, and used the standard deduction instead of $20,520 of itemized deductions. Its bracket math then gave only $33,544 on $565,375 of taxable income, which is far below what the 9.3% bracket produces. It also subtracted an unphased $892 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its bracket computation gave $29,423 on $495,603 of taxable income, but the 2026 MFJ schedule with 9.3% on the top slice produces about $39,000 there. It also used the standard deduction instead of the larger $20,520 of itemized deductions."
us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $19,285 SSDI as non-taxable, but SSDI is a Social Security benefit taxed under IRC §86, so $13,784.12 belongs in AGI. It also used the outdated 2024 $14,600 standard deduction instead of the 2026 $16,100. Its taxable income was therefore $20,680 instead of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Its own derivation was nearly right: AGI $49,064, taxable income $32,964 and tax of about $3,713, off only because its 10% bracket ended at $12,150 instead of $12,400. It then swapped in a guessed $16,600 deduction and finally submitted $2,386, a number its reasoning never produced."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It taxed the full 85% cap ($16,392) of the SSDI. §86 taxes the lesser of that cap and 0.85×($44,922.50 − $34,000) + $4,500 = $13,784, so this overstated AGI by $2,608. It then abandoned its own $4,021 result and submitted an unsupported $1,858."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It derived the correct figures (taxable SS $13,784, AGI $49,064, taxable income $32,964, tax about $3,708 using the $12,400 10% bracket). It then submitted $2,534, a value that contradicts its own completed calculation."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed that provisional income above $34,000 makes a flat 85% of benefits ($16,392.25) taxable. It skipped the §86 lesser-of formula, which gives $13,784.12, and it wrongly added the tax-exempt pension to provisional income. It also used the 2025 pre-OBBBA $15,000 standard deduction and $11,925 bracket instead of the 2026 $16,100 and $12,400."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It replaced the exact §86 result of $13,784 with a made-up 'worksheet approximation' of $15,650 and used the 2025 $11,925 bracket, reaching $3,941. It then submitted $1,611, which its reasoning does not support."
-us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it estimated the 2026 single standard deduction at $15,400 instead of the OBBBA-indexed $16,100. It also ended the 10% bracket at $12,250 instead of $12,400, which overstated taxable income by $700."
-us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used a $15,300 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. Taxable income came out at $33,764 instead of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted the full 85% cap ($16,392.25) as taxable SS. The §86 lesser-of formula, 0.85×($44,922.50 − $34,000) + $4,500, gives $13,784.12. It also used a guessed $15,400 standard deduction and a $12,225 bracket instead of $16,100 and $12,400."
-us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied pre-TCJA rules for 2026: an $8,100 standard deduction plus a $5,300 personal exemption, and a 15% bracket. OBBBA (P.L. 119-21) made the TCJA structure permanent, so the correct figures are a $16,100 deduction, no exemption, and 10%/12% brackets."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used an estimated $15,700 standard deduction instead of $16,100. It also taxed only $11,925 at 10% (the 2025 bracket) instead of $12,400, which pushed the tax up to $3,765."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied a head-of-household standard deduction with the age-65-or-older add-on and subtracted nonrefundable credits. The Head is a 56-year-old single filer, so the deduction is $16,100 and no nonrefundable credit applies, which leaves $3,707.70 of tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly applied pre-TCJA rules for 2026, with a standard deduction plus a personal exemption. OBBBA made the $16,100 single standard deduction, the zero personal exemption and the 10%/12% brackets permanent, and those give $3,707.70."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied TCJA-sunset rules: an $8,300 standard deduction plus a $5,150 personal exemption and pre-TCJA brackets. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent for 2026."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its explanation subtracts nonrefundable credits, but none apply here. The credit for the elderly or disabled is fully phased out: its $5,000 base is reduced by $5,501 of nontaxable SS and by half of AGI over $7,500. Its $3,212 is $496 below the zero-credit result of $3,707.70."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 was correct, but its $3,807.18 is $99.48 too high. That gap matches a standard deduction of about $15,300 instead of the 2026 $16,100, which put taxable income near $33,760 instead of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation reaches the correct $3,707.70 from AGI of $49,064.13, the single standard deduction and the 2026 brackets. It nevertheless submitted $3,744 in the value field, contradicting its own stated result."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used a $15,350 standard deduction instead of $16,100 and ended the 10% bracket at $12,200 instead of $12,400. Taxable income came out at $33,714 and tax at $3,801.68."
-us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used 2024 parameters (a $14,600 standard deduction and an $11,600 10% bracket) instead of the 2026 $16,100 and $12,400. It also added the $284 tax-exempt pension to provisional income, which inflated taxable SS to $14,025.53 instead of $13,784.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It applied only the first-tier 50% formula, min(0.85×SS, 0.5×(PI − $25,000)) = $9,961.25. Because provisional income exceeds $34,000, the second tier applies: 0.85×(PI − $34,000) + $4,500 = $13,784.12. It also used the 2025 $15,750 deduction and a $12,000 bracket instead of $16,100 and $12,400."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction offsets the income and reported $0. The $16,100 deduction against $49,064.12 of AGI leaves $32,964.12 taxable, which produces $3,707.70 of tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that nonrefundable credits reduce the tax to $0, but no nonrefundable credit applies. The single filer has no dependents, and the elderly-or-disabled credit is fully phased out by $49,064 AGI and $5,501 of nontaxable SS. The $3,707.70 of tax on $32,964.12 of taxable income stands in full."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used the 2025 OBBBA standard deduction of $15,750 instead of the inflation-indexed 2026 amount of $16,100. That overstated taxable income by $350 and tax by $42."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,572 implies it counted the full 85% cap ($16,392) of the SSDI as taxable. The §86 lesser-of formula caps taxable SS at $13,784.12, which gives taxable income of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $33,206 is exactly what results from adding the $284 tax-exempt pension to provisional income, which gives taxable SS of $14,025.53. §86 modified AGI adds back only tax-exempt interest, so taxable SS is $13,784.12 and taxable income is $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,736.66 equals $1,240 + 12% × ($33,205.53 − $12,400), which means it added the $284 tax-exempt pension to provisional income and got $14,025.53 of taxable SS. The pension is not tax-exempt interest under §86(b)(2), so the correct taxable SS is $13,784.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took taxable SS as the full 85% cap of $16,392, giving AGI of $51,672. It should have applied the §86 lesser-of formula, 0.85×($44,922.50 − $34,000) + $4,500 = $13,784.12, which gives AGI of $49,064.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $2,444 implies taxable income of about $22,400, roughly $10,500 below the correct $32,964.12. That shortfall matches leaving out most of the $13,784.12 of taxable SSDI required by §86."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it projected post-TCJA-sunset 2026 law: an $8,535 standard deduction, a $5,443 personal exemption and 10%/15% brackets. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied post-TCJA-sunset rules: an $8,496 standard deduction, a $5,419 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, with a $16,100 deduction and 10%/12% rates."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it treated 2026 as post-TCJA-sunset, with a $5,450 personal exemption, an $8,550 standard deduction and a 15% bracket. OBBBA permanently extended the TCJA rates and the $16,100 single deduction."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted the full 85% cap ($16,392) as taxable SS instead of the §86 lesser-of amount, $13,784.12. It also used a guessed $15,420 standard deduction and a $12,259 bracket instead of $16,100 and $12,400."
-us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 was correct, but it applied a 2026 'deduction/exemption' pair under pre-TCJA law, and its $4,708.37 matches a roughly $13,650 deduction-plus-exemption taxed with a 15% bracket. OBBBA-permanent law uses a $16,100 deduction with 10%/12% brackets, which gives $3,707.70."
-us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It set AGI to the $35,280 pension alone and left out the $13,784.12 of SSDI that §86 makes taxable. It also misstated first-bracket tax as $1,540 instead of $1,240 on the first $12,400."
-us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added the $284 tax-exempt pension to combined income, and it summed the components to $45,406 when they total $45,206.50. That produced taxable SS of $14,196 instead of $13,784.12, and AGI and tax came out $412 and $49 too high."
-us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"In the second-tier §86 formula it used $6,000, the married-filing-jointly figure, instead of the single filer's min($4,500, 50% of benefits). It also added the tax-exempt pension to combined income, which gave $15,525.53 of taxable SS instead of $13,784.12. It then used the 2025 $15,750 deduction instead of $16,100."
-us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left the $13,784.12 of taxable SSDI out of income and used a nonexistent 12.5% rate, computing $2,177. It then submitted $4,202, which contradicts its own arithmetic."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Treated SSDI as entirely non-taxable. Because provisional income of $44,922.50 is above the $34,000 single threshold, $13,784.12 of the SSDI is taxable. It also used the 2024 $14,600 standard deduction instead of the 2026 $16,100 deduction."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Assumed taxable SSDI was about $0 because other income was 'low.' The $35,280 pension raises provisional income to $44,922.50, which makes $13,784.12 taxable and AGI $49,064.12. It therefore taxed only about $19,180 instead of $32,964.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Correctly derived taxable income of $32,964 but used a $12,150 top for the 10% bracket instead of $12,400, which gave $3,713. It then switched to a $16,600 deduction and submitted $2,386, a figure that none of its own calculations support."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser of the cap and the formula amount ($13,784.12), which overstated AGI at $51,672 and gave $4,021. It then submitted $1,858, which none of its own figures support."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Correctly computed taxable SS ($13,784), taxable income ($32,964) and tax of about $3,708 using the $12,400 10% bracket. It then submitted $2,534, which contradicts its own derivation."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Taxed the full 85% cap ($16,392.25) of SSDI without applying the lesser-of formula, which gives $13,784.12, and added the $284 tax-exempt pension to provisional income. It also fell back on the 2025 $15,000 deduction and the $11,925 10% bracket instead of 2026's $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Set up the correct taxable-SS formula but replaced its result with an 'approximation' of $15,650 instead of $13,784.12, and used the 2025 $11,925 10% bracket, reaching $3,941. It then submitted $1,611, which its own work does not support."
+us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a projected $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100, and a $12,250 top for the 10% bracket instead of $12,400. This overstated taxable income at $33,664."
+us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but subtracted a $15,300 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. This produced taxable income of $33,764 and tax of $3,808."
+us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392.25) as taxable SS instead of the lesser formula amount of $13,784.12, and used a $15,400 deduction and a $12,225 10% bracket instead of 2026's $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA rules: an $8,100 standard deduction, a $5,300 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 deduction, no personal exemption, and 10%/12% brackets that break at $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a $15,700 standard deduction instead of $16,100 and the 2025 $11,925 10% bracket instead of 2026's $12,400. This overstated taxable income at $33,364 and the tax."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Applied the head-of-household, age-65-or-older standard deduction and claimed nonrefundable credits. The filer is a 56-year-old single person with no dependents, who gets only the $16,100 single deduction and has no credits."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Modeled 2026 under pre-TCJA rules, with a personal exemption and the smaller standard deduction. OBBBA made the TCJA structure permanent, so the correct calculation is the $16,100 deduction and 10%/12% brackets on $32,964.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but subtracted a pre-TCJA $8,300 standard deduction and a $5,150 personal exemption. Under 2026 law the deduction is $16,100 and there is no exemption, and the 10%/12% brackets apply."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 equals the tax when only 50% of the SSDI ($9,642.50) is taxable (AGI $44,922.50). Because provisional income is above the $34,000 threshold, the 85% tier makes $13,784.12 taxable."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064.13, but its $3,807.18 matches a $15,300 standard deduction with a $12,225 10% bracket. The 2026 values are $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation reaches $3,707.70 from AGI of $49,064.13, the 2026 single deduction and the 2026 brackets, but it submitted $3,744. The submitted value contradicts its own correct calculation."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a $15,350 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. This overstated taxable income at $33,714."
+us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Added the $284 tax-exempt pension to provisional income, which raised taxable SS to $14,025.53 instead of $13,784.12. It also used the 2024 $14,600 deduction and $11,600 10% bracket instead of 2026's $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Applied only the 50% tier, 0.5 × ($44,922.50 − $25,000) = $9,961.25, and ignored the 85% tier for provisional income above $34,000, which makes $13,784.12 taxable. It also used a $15,750 deduction and a $12,000 10% bracket instead of $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction wipes out the tax. AGI is $49,064.12, so after the $16,100 deduction there is $32,964.12 of taxable income, taxed at 10%/12% with no nonrefundable credits."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Concluded that the deduction and credits reduce tax to $0. The $35,280 pension plus $13,784.12 of taxable SSDI leaves $32,964.12 taxable after the $16,100 deduction, and no nonrefundable credit applies."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used the 2025 $15,750 single standard deduction instead of the 2026 $16,100. This overstated taxable income by $350 ($33,314)."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,572 shows it taxed the 85% cap ($16,392) of SSDI instead of the lesser formula amount of $13,784.12, which overstated AGI by $2,608."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $33,206 is $241 above $32,964.12. That gap equals 0.85 × $284, so it added the tax-exempt pension to provisional income, which the taxable-SS formula does not count."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,736.66 implies taxable income of $33,205.50, which is $241.38 above $32,964.12. That gap equals 0.85 × $284, so it counted the tax-exempt pension in provisional income when computing taxable SS."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser of the cap and 0.85 × ($44,922.50 − $34,000) + $4,500 = $13,784.12, which overstated AGI at $51,672."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $2,444 implies taxable income of about $22,400, roughly $10,500 too low. That fits counting only a small part of the SSDI as taxable instead of the $13,784.12 required when provisional income is above $34,000."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but assumed the TCJA had expired: an $8,535 deduction, a $5,443 personal exemption and a 15% bracket. 2026 law keeps the $16,100 deduction, no exemption, and 10%/12% brackets."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a pre-TCJA $8,496 deduction plus a $5,419 personal exemption and a 15% second bracket. 2026 law has a $16,100 deduction, no exemption, and a 12% bracket above $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but treated 2026 as post-TCJA-sunset, with a $5,450 exemption, an $8,550 deduction and a 15% bracket. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent, so taxable income is $32,964.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser formula amount of $13,784.12, and used a $15,420 deduction and a $12,259 10% bracket instead of 2026's $16,100 and $12,400."
+us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Gave no value or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064.13, but applied a pre-TCJA 'deduction/exemption' and bracket structure that yields $4,708.37. 2026 law uses the $16,100 deduction with no personal exemption and 10%/12% brackets."
+us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Left out the taxable SSDI ($13,784.12) and set AGI equal to the $35,280 pension alone. It also miscomputed the 10%-bracket tax as $1,540 instead of $1,240."
+us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Added the $284 tax-exempt pension to provisional income and mis-added the total as $45,406 instead of $45,206.50. This inflated taxable SS to $14,196 instead of $13,784.12 and AGI to $49,476."
+us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Used $6,000, the joint-filer amount, instead of the single filer's $4,500 in the 85%-tier formula, and added the $284 tax-exempt pension to provisional income, which gave $15,525.53 of taxable SS instead of $13,784.12. It also used a $15,750 deduction and a $12,150 10% bracket."
+us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Left out the $13,784.12 of taxable SSDI, used a non-existent 12.5% rate and a $15,750 deduction to get $2,177, and then submitted $4,202, which its own calculation does not support."
us,scenario_007,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_007,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly noted that the head is under 65, but it assumed the 24-month SSDI waiting period was met because of the ""constant full-year status"" rule. A constant status over one tax year shows at most 12 months of receipt. No listed fact shows 24 months of entitlement, and the prompt treats unlisted status inputs as false, so neither the disability pathway nor the age pathway applies."
us,scenario_007,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model said SSDI income by itself confers Medicare eligibility and skipped the 24-month entitlement requirement for disability-based Medicare. No listed fact satisfies that requirement, and at 56 the head fails the age-65 test, so the answer is not eligible."
@@ -355,88 +353,90 @@ us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disabili
us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first recognized that no wage was listed but then asserted $1,516.30 of employee Social Security and Medicare tax on unspecified “applicable taxable wages.” The listed Social Security disability and private pension payments are not employee FICA wages, and the 50 weekly hours cannot create wage income when the unlisted wage rate and annual wages are zero."
us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract."
us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of Idaho taxable income. It then applied the obsolete 5.695% rate from the first dollar ($1,092) and raised that to $1,571 with an unsupported 'inflation adjustment'. The frozen reference applies 5.3% only to the $14,260 above the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It derived the frozen reference's $19,180 of taxable income but taxed all of it at 5.3%. It left out Idaho's 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920 and so taxes only $14,260, for $755.78. The exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It invented a $41,110 Idaho 'retirement income exclusion' for the private pension. Idaho's retirement benefits deduction covers only qualifying public and military retirement pay, and only at age 65 (or 62 if disabled). This head is 56 with a private pension, so the $35,280 stays taxable and gives $755.78 of tax in the frozen reference; the exclusion's corrected value, which also subtracts the $2,080 of premiums and uses the $4,811 zero-rate band that c_id_hold_2025 keeps for 2026, is $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of Idaho taxable income. It then used the outdated 5.695% rate on the full amount and raised the result to $1,379 with an arbitrary 'conformity' adjustment. The frozen reference taxes $19,180 − $4,920 = $14,260 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used the 2025 standard deduction of $15,750 instead of 2026's $16,100, and the old 5.695% rate above a $4,673 threshold, giving $846. It then raised that to $1,300 without any basis. The frozen reference's figure is 5.3% × ($19,180 − $4,920) = $755.78, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It said it would apply 5.3% above a roughly $4,700 exempt amount, but its $1,112 equals 5.8% × $19,180. That means it actually applied the superseded 5.8% rate to all taxable income with no zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $109 more at 5.3% than the frozen reference, giving $761.56 against its $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It explicitly rejected Idaho's full subtraction of federally taxable Social Security, and it overstated taxable Social Security at 85% ($16,392 instead of $13,784). It also used a $15,000 standard deduction, a $4,950 personal exemption that doesn't exist, and the 5.695% rate, and it subtracted Idaho's grocery credit, which is refundable and so doesn't belong in a before-refundable-credits figure."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used a $14,600 standard deduction instead of the 2026 amount of $16,100 and applied 5.695% to the full $20,680, giving $1,178. It then raised that to $1,800 with no stated computation. It never applied the 5.3% rate above the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of taxable income but applied 5.3% to all of it. The frozen reference taxes only the $14,260 above the engine's projected $4,920 zero-rate band, giving $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and taxes only the income above the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It did not apply Idaho's full subtraction of the $13,784 of federally taxable Social Security, and it used a $14,000 standard deduction instead of $16,100. It then applied 5.695% to $35,064 with no zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It correctly subtracted Social Security but used a $15,300 standard deduction instead of the 2026 amount of $16,100. It then taxed all $19,980 at the repealed 5.8% rate. The frozen reference taxes $19,180 − $4,920 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, taxes $17,100 − $4,811 at 5.3%, after the $2,080 premium subtraction and with the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It used a $15,400 standard deduction instead of $16,100 and applied the outdated 5.695% rate to all $19,880. It never excluded Idaho's zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference) or used the 5.3% rate."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,100 standard deduction instead of the federal-conforming 2026 single deduction of $16,100, which overstated taxable income at $27,180. It then taxed all of it at 5.3% without the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It used a $15,700 standard deduction instead of $16,100 and applied the repealed 5.8% rate to all $19,580. It left out the 2026 5.3% rate and the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It treated the disability income as taxable for Idaho. Idaho subtracts all federally taxable Social Security, including SSDI, leaving only the $35,280 pension. In the frozen reference, after the $16,100 deduction and the engine's projected $4,920 zero-rate band, tax is 5.3% × $14,260 = $755.78, about half of its $1,582; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It correctly excluded Social Security, but $1,246 matches roughly $21,500–$23,500 taxed at a flat rate from the first dollar. That means it understated the $16,100 standard deduction and left out the $4,920 zero-rate band. The frozen reference's computation is 5.3% × $14,260 = $755.78, above the engine's projected $4,920 zero-rate band; the exclusion's corrected value is 5.3% × $12,289 = $651.32, after the $2,080 premium subtraction and the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It subtracted only $13,450 of standard deduction and 'personal exemption' from $35,280, reaching $21,830, when the 2026 standard deduction alone is $16,100. It then applied the outdated 5.695% rate to all of it with no zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"$3,371 is more than 6% of total gross income, which means it taxed roughly all $54,565 of income. It never applied the Social Security subtraction, the $16,100 standard deduction, the $4,920 zero-rate band, or the 5.3% rate."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It correctly reached $35,280 after the Social Security subtraction, but its $1,115.08 equals 5.695% × $19,580. That means it used a $15,700 standard deduction instead of $16,100 and the outdated 5.695% rate on all taxable income, with no $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation computes 5.695% × $19,180 = $1,092.30, using the outdated rate with no zero-rate band, but it submitted an unrelated $1,894. Neither figure applies 5.3% only to income above Idaho's zero-rate band: the frozen reference taxes the $14,260 above the engine's projected $4,920, and the exclusion's corrected value taxes the $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It used a $15,350 standard deduction instead of $16,100 and taxed all $19,930 at the repealed 5.8% rate. It left out the 2026 5.3% rate and the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It claimed Idaho's retirement benefits deduction removes the $35,280 private pension because the head is disabled. That deduction covers only qualifying public and military retirement pay for recipients aged 65+ (or 62+ if disabled), so it doesn't apply to a 56-year-old's private pension, and $19,180 remains taxable."
-us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It used the 2025 standard deduction of $15,750 instead of $16,100, taxed all $19,530 at 5.3% with no $4,920 zero-rate band, and added a $10 permanent building fund tax. That fund tax is not part of Idaho income tax before refundable credits."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed the standard deduction and exclusions fully offset income. Idaho income after the Social Security subtraction is $35,280, which exceeds the $16,100 standard deduction plus the zero-rate band. The frozen reference, with the engine's projected $4,920 band, leaves $14,260 taxed at 5.3%; the exclusion's corrected value also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026, leaving $12,289 taxed at 5.3%, for $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It returned zero tax without any computation. The $35,280 private pension remains in Idaho income. In the frozen reference, after the $16,100 standard deduction and the engine's projected $4,920 zero-rate band, $14,260 is taxed at 5.3%, for $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used a guessed $5,000 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,180 at 5.3%, giving $751.54, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"$1,885 equals 5.3% × about $35,570, which is federal AGI with 85% of the SSDI included, minus $16,100. That means it skipped Idaho's full subtraction of taxable Social Security and also left out the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 but applied 5.3% to the entire amount. It left out Idaho's zero-rate band: the frozen reference excludes the engine's projected $4,920, leaving $14,260 taxable, and the exclusion's corrected value, after the $2,080 premium subtraction, excludes the $4,811 that c_id_hold_2025 keeps for 2026, leaving $12,289."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"$1,759.89 equals 5.3% × about $33,200, which is federal taxable income. That means it never applied Idaho's subtraction of the federally taxable Social Security and also left out the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It subtracted the head's $2,080 of health insurance premiums from Idaho income, the Idaho subtraction (Idaho Code 63-3022P) that the engine leaves out; the frozen reference's Idaho income of $35,280 has only the Social Security subtraction. It also used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Its $651.32 is the exclusion's corrected value."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It computed taxable income of $35,572 from a federal AGI that included 85% of the SSDI, and never applied Idaho's full subtraction of taxable Social Security. It then taxed everything at 5.3% with no $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It used an approximate $4,990 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,190 at 5.3% and gave $752, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $14,369 at 5.3%, against $14,260 in the frozen reference."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"$1,477 implies roughly $26,000–$28,000 taxed at a flat rate. That means it used a TCJA-sunset standard deduction of about $8,000 instead of the permanent 2026 amount of $16,100, and left out the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It assumed an $8,535 standard deduction, as if TCJA had expired, instead of the permanent 2026 single deduction of $16,100. It then applied the outdated 5.695% rate to all $26,745 with no zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It used a standard deduction of about $8,477, based on a TCJA sunset, instead of $16,100. It then taxed all $26,803 at 5.695% instead of taxing at 5.3% only the income above the zero-rate band: $14,260 above the engine's projected $4,920 in the frozen reference, or $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction, in the exclusion's corrected value."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It assumed the standard deduction reverted to the post-sunset $8,550. OBBBA made the higher deduction permanent, at $16,100 for 2026. It also applied 5.3% to all $26,730 without the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Its $36,252 taxable income comes from a federal AGI that included 85% of the SSDI, with no Idaho Social Security subtraction and a $15,420 deduction. It then applied the repealed 5.8% rate from the first dollar."
-us,scenario_007,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed federal taxable income of $32,964 directly and never subtracted the $13,784 of federally taxable Social Security that Idaho excludes. It also left out the $4,920 zero-rate band."
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the answer is missing rather than wrong."
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It subtracted only $13,600 of deduction/exemption from $35,280 and then used an invented $2,500 zero-rate band, and the two errors happened to land on $19,180 × 5.3%. The frozen reference subtracts the $16,100 standard deduction to reach $19,180 and then excludes the engine's projected $4,920 zero-rate band; the exclusion's corrected value also subtracts the $2,080 of premiums, for $17,100, and excludes the $4,811 band that c_id_hold_2025 keeps for 2026."
-us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It stated that Idaho has no individual income tax. Idaho does: it taxes income above a zero-rate band at a flat 5.3%. With the engine's projected $4,920 band the frozen reference is $755.78; the exclusion's corrected value, with the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026 and the $2,080 premium subtraction, is $651.32."
-us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It derived the frozen reference's $19,180 of Idaho taxable income but applied 5.3% to all of it. It left out the 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It treated the full $19,285 of SSDI as taxable in federal AGI and in Idaho, instead of applying Idaho's full Social Security subtraction. It used a $14,600 deduction and the 5.8% rate, subtracted the grocery credit even though it is refundable, and then made an arbitrary cut to reach $1,707.48."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It started from a base that already had the $15,750 deduction applied, then subtracted a second, invented $5,200 'Idaho standard deduction'. It also applied the repealed graduated brackets instead of the 5.3% flat rate above the zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference)."
-us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly picked the Idaho Grocery Credit as the refundable credit, but used the outdated $120 under-65 amount. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Idaho has no refundable individual income tax credits. That overlooks the Idaho Grocery Credit, which is refundable and worth $155 in 2026 to this under-65 resident, who received no SNAP."
-us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning cites a $120 under-65 Grocery Credit, yet it submitted $140, which is the old 65-and-over amount. Neither is the 2026 under-65 amount of $155."
-us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Claimed the Grocery Credit is nonrefundable when claimed on the income tax return. It is a refundable credit on the Idaho return, so the full $155 counts as a refundable state credit."
-us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Recognized that the Grocery Credit works like a refundable credit, then reclassified it as a nonrefundable 'other item' and zeroed it. The credit is refundable and worth $155 in 2026 to a resident under 65."
-us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Reasoned that a filer with Idaho tax liability uses the Grocery Credit up as a nonrefundable offset, leaving $0 refundable. The Grocery Credit is refundable whatever the liability, so the full amount counts. The model also used the outdated $120 figure, not the 2026 amount of $155."
-us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Treated the Idaho Grocery Credit as nonrefundable against tax liability. It is a refundable credit, worth $155 to this under-65 resident in 2026."
-us,scenario_007,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Correctly treated the Grocery Credit as refundable, but used the outdated $120 per-person amount. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Invented a roughly $22,500 income phaseout for the Idaho Grocery Credit. The credit has no income phaseout, so this resident receives the full $155 regardless of pension and SSDI income."
-us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit for one person, but used the outdated $120 amount. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used $100, the Grocery Credit amount from before 2023, for a filer under 65. The 2026 amount is $155."
-us,scenario_007,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Applied the pre-2023 $100 Grocery Credit amount. The 2026 amount for a resident under 65 is $155.
-us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Identified the refundable Grocery Credit, but used the outdated $120 per-member amount. The 2026 amount for this under-65 member is $155."
-us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that the household qualifies for no refundable Idaho credits. That misses the Idaho Grocery Credit: $155 in 2026 for a full-year resident under 65 without SNAP, refundable and with no income test."
-us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Got eligibility right (under 65, no SNAP), but used the outdated $120 Grocery Credit amount. The 2026 amount is $155."
-us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,Applied the outdated $120 Grocery Credit for a single filer under 65. The 2026 amount is $155.
-us,scenario_007,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Answered $0 without reasoning, so it never applied the Idaho Grocery Credit. That credit is refundable and gives this under-65 resident, who received no SNAP, $155 in 2026."
-us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Identified the refundable Grocery Credit per resident, but used the outdated $120 amount. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly found the head eligible for the refundable Grocery Credit, but used $120. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Applied the pre-2023 $100 Grocery Credit amount for a resident under 65. The 2026 amount is $155.
-us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Falsely asserted that Idaho repealed the Grocery Credit. The credit remains in force and is refundable, worth $155 in 2026 to this under-65 resident, who received no SNAP."
-us,scenario_007,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Correctly applied the refundable Grocery Credit with no income limit, but used the pre-2023 $100 amount. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable Idaho credit, missing the Grocery Credit. Every full-year resident without SNAP receives it, and it is worth $155 in 2026 to this filer under 65."
-us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed refundable state credits require a special qualifying condition, such as having children. The Idaho Grocery Credit needs only Idaho residency and no SNAP receipt, and pays $155 in 2026 to a filer under 65."
-us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Claimed the household's income is above a phaseout for the Idaho Grocery Credit. The credit has no income phaseout, so the full $155 under-65 amount applies."
-us,scenario_007,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Concluded that no Idaho refundable credit applies. It overlooked the Grocery Credit, which is refundable and worth $155 in 2026 to this full-year resident under 65."
-us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit for a full-year resident under 65, but used the pre-2023 $100 amount. The 2026 amount is $155."
-us,scenario_007,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,Applied the outdated $100 Grocery Credit amount. The 2026 amount for a resident under 65 is $155.
-us,scenario_007,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Correctly found the full-year resident eligible for the refundable Grocery Credit, but used $120. The 2026 amount for a filer under 65 is $155."
-us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Concluded that no state refundable credit applies to this household composition. The Idaho Grocery Credit applies to any resident without SNAP and pays $155 in 2026 to this filer under 65.
-us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_refundable_credits, so there is no answer to score against the $155 Grocery Credit."
-us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit as the only applicable Idaho refundable credit, but used the outdated $120 amount. The 2026 amount for a person under 65 is $155."
-us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Asserted that Idaho has no state income tax. Idaho does levy an individual income tax, and its return includes the refundable Grocery Credit, worth $155 in 2026 to this resident under 65."
-us,scenario_007,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,Guessed an inflation-adjusted Grocery Credit of about $140. The credit is not indexed; the statutory 2026 amount for a resident under 65 is $155.
-us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Dismissed Idaho's refundable credits as insignificant and answered $0. It skipped the Grocery Credit, which pays this under-65 resident, who received no SNAP, $155 in 2026."
-us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Concluded that no Idaho refundable credit applies. It missed the Grocery Credit, which is refundable, has no income test and pays $155 in 2026 to a resident under 65."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It correctly got Idaho taxable income of $19,180. It then applied the old 5.695% rate instead of 2026's 5.3% and left out the $4,920 zero-rate amount. After computing about $1,092, it raised the result to $1,571 for no stated reason, when 0.053 × ($19,180 − $4,920) = $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It correctly derived $19,180 of taxable income and the 5.3% rate. However, it taxed the whole $19,180 and left out Idaho's $4,920 single-filer zero-rate amount, which would have taken $260.76 off the tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It made up a $41,110 retirement-income exclusion for private pensions. Idaho's retirement-benefits deduction covers only certain public, military, and similar pensions, and only from age 65 (62 if disabled), so it does not apply to a 56-year-old's private pension. The $35,280 pension stays taxable, giving $19,180 of taxable income after the $16,100 standard deduction and $755.78 of tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"Its $0 answer treats the pension as fully sheltered, but no Idaho deduction offsets it. Idaho income of $35,280 minus the $16,100 standard deduction leaves $19,180, and 5.3% on the amount above $4,920 is $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It correctly reached $19,180 of Idaho taxable income. It then used 5.695% instead of 5.3%, left out the $4,920 zero-rate amount, and raised its own $1,092 figure to $1,379 based on an unexplained 'conformity estimate'."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used a $15,750 standard deduction instead of 2026's $16,100, the 5.695% rate instead of 5.3%, and a $4,673 zero-rate amount instead of $4,920. It then raised its own $846 result to $1,300 without any computation."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It stated the right method: 5.3% applied to $19,180 above roughly $4,700. However, the $1,112 it submitted equals 5.8% of the full $19,180, with no zero-rate amount and the wrong rate. Applying its own stated method with the correct $4,920 zero-rate amount gives $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It used 2025's $4,811 zero-rate amount instead of 2026's inflation-adjusted $4,920. That taxed an extra $109 at 5.3%, giving $761.56 instead of $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It wrongly decided Idaho has no Social Security subtraction and taxed $16,392 of disability benefits. Idaho subtracts all federally taxable Social Security, which here is $13,784 under the provisional-income formula, not a flat 85%. It also made up a $4,950 personal exemption, used a $15,000 standard deduction and the 5.695% rate, and subtracted the grocery credit, which Idaho pays as a refundable credit."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used a $14,600 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount. It then raised its own $1,178 estimate to $1,800 for no reason."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It correctly got $19,180 of taxable income and the 5.3% rate. It then applied 5.3% to the whole amount and left out the $4,920 single-filer zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It never applied Idaho's subtraction of the $13,784 of federally taxable Social Security and taxed the full $49,064 federal AGI. It also used a $14,000 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It correctly subtracted Social Security. It then used a $15,300 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole amount without the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It correctly started from $35,280. It then used a $15,400 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,100 standard deduction instead of 2026's $16,100, which overstated taxable income by $8,000. It also applied 5.3% to the whole amount without the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It used a $15,700 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole amount without Idaho's $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It counted 'taxable disability income' in Idaho income even though Idaho subtracts all federally taxable Social Security. Its $1,582 answer corresponds to a taxable base of roughly $28,000–$30,000, not $19,180. It also left out the $4,920 zero-rate amount on top of 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It correctly excluded Social Security. However, its $1,246 answer matches a flat rate (5.695% or 5.8%) applied to about $21,500–$21,900 with no zero-rate amount, which means the standard deduction was too small and the $4,920 zero-rate amount was missing. The correct result is 0.053 × ($19,180 − $4,920) = $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It took a combined $13,450 standard deduction and personal exemption, but Idaho has no personal exemption and its 2026 standard deduction is $16,100. That gave $21,830 of taxable income, which it taxed at 5.695% instead of 5.3%, without the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"Its $3,371 answer requires a taxable base above the household's entire $54,565 gross income at any Idaho rate. That means it neither subtracted Social Security nor applied the standard deduction or the zero-rate amount. The correct base is $19,180, taxed at 5.3% above $4,920."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It correctly got Idaho AGI of $35,280. Its $1,115.08 equals 5.695% of $19,580, so it used a $15,700 standard deduction instead of $16,100, the old 5.695% rate instead of 5.3%, and no $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation computes $1,092.30, which is 5.695% of $19,180, with the wrong rate and no $4,920 zero-rate amount. It then submitted $1,894, which contradicts its own explanation. The correct figure is 0.053 × ($19,180 − $4,920) = $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It used a $15,350 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole $19,930 without the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It applied Idaho's retirement-benefits deduction to the $35,280 private pension. That deduction covers only qualifying public, military, and similar pensions, and only for filers aged 65 or older (62 if disabled), and this head is 56. Without it, taxable income is $19,180 and the tax is $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It used a $15,750 standard deduction instead of $16,100, which gave $19,530 instead of $19,180. It then applied 5.3% to the full amount without the $4,920 zero-rate amount and added a $10 permanent building fund tax, which is not part of this output."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed the standard deduction fully offsets income, but the $35,280 taxable pension is well above the $16,100 standard deduction plus the $4,920 zero-rate amount. That leaves $14,260 taxed at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It returned $0 without showing any calculation. Idaho income of $35,280 minus the $16,100 standard deduction leaves $19,180, and 5.3% on the amount above $4,920 is $755.78, so the pension income is taxable."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used a $5,000 zero-rate amount instead of 2026's $4,920. That undertaxed $80 at 5.3% and gave $751.54 instead of $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"Its $1,885 equals 5.3% of about $35,570, which is federal AGI with 85% of Social Security included ($51,672) minus $16,100. That means it never applied Idaho's full subtraction of taxable Social Security, and it also left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It correctly derived $19,180 of taxable income and the 5.3% rate. It then taxed the full amount and left out Idaho's $4,920 single-filer zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Its $1,759.89 equals 5.3% of about $33,205, which is federal taxable income (federal AGI minus $16,100). That means it left out Idaho's subtraction of the $13,784 of federally taxable Social Security as well as the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It subtracted $2,080 of health premiums that do not reduce Idaho taxable income here. Idaho AGI of $35,280 minus only the $16,100 standard deduction gives $19,180. It also used 2025's $4,811 zero-rate amount instead of 2026's $4,920."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"Its $35,572 taxable income is $51,672 of federal AGI (with 85% of Social Security included) minus $16,100, so it never applied Idaho's subtraction of federally taxable Social Security. It also used a flat 85% inclusion instead of the $13,784 formula amount and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It used an approximate $4,990 zero-rate amount instead of 2026's $4,920. That undertaxed $70 at 5.3% and gave $752 instead of $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It carried forward 2025's $4,811 zero-rate amount instead of 2026's inflation-adjusted $4,920. That taxed an extra $109 at 5.3% and gave $761.56."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"Its $1,477 equals 5.3% of about $27,870, which means a standard deduction far below 2026's $16,100 and no $4,920 zero-rate amount. The correct calculation is 0.053 × ($35,280 − $16,100 − $4,920) = $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It used a pre-TCJA-style $8,535 standard deduction instead of 2026's $16,100, which the One Big Beautiful Bill Act made permanent. It also applied the old 5.695% rate instead of 5.3% and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It used a roughly $8,480 'conforming' standard deduction instead of $16,100, which gave $26,803 instead of $19,180. It then applied 5.695% instead of 5.3% and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It assumed the TCJA standard deduction expired and used the post-sunset $8,550. The One Big Beautiful Bill Act made the higher deduction permanent, and it is $16,100 for 2026. It also applied 5.3% to the full $26,730 without the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Its $36,252 base is federal AGI with 85% of Social Security included ($51,672) minus $15,420, so it never applied Idaho's subtraction of taxable Social Security. It also used the old 5.8% rate and left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed the federal taxable income of $32,964 directly. That skips Idaho's subtraction of the $13,784 of federally taxable Social Security, which brings the base down to $19,180, and it also left out the $4,920 zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It used a $13,600 deduction instead of $16,100 and a $2,500 zero-rate amount instead of $4,920. These errors offset each other enough that 5.3% ended up applied to the whole $19,180, overstating tax by the $4,920 zero-rate portion."
+us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It claimed Idaho has no individual income tax. Idaho taxes individual income at a flat 5.3% above a $4,920 zero-rate amount, which here yields $755.78 on $19,180 of taxable income."
+us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It correctly got $19,180 of taxable income and the 5.3% rate. It then applied the rate to the full amount without Idaho's $4,920 single-filer zero-rate amount."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It treated all $19,285 of Social Security disability benefits as taxable and never applied Idaho's subtraction of federally taxable Social Security. It also used a $14,600 standard deduction and the 5.8% rate, subtracted the grocery credit (refundable in Idaho), and then cut the result to $1,707.48 for no stated reason."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It took a second $5,200 standard deduction from a base that had already been reduced by the federal deduction. It also applied graduated brackets that Idaho repealed in 2023. The correct calculation applies the single flat 5.3% rate above $4,920 to $19,180."
+us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly picked the refundable Idaho Grocery Credit as the only refundable state credit, but used the old $120 per-person amount. HB 231 raised the under-65 amount to $155 starting in tax year 2025, so the 2026 credit is $155."
+us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said Idaho has no refundable individual income tax credits and missed the Idaho Grocery Credit, which is refundable. That credit gives this 12-month-qualifying resident $155 for 2026."
+us,scenario_007,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,Said no refundable state credits apply and missed the refundable Idaho Grocery Credit. That credit is $155 per person for 2026 for a full-year resident who is not on SNAP.
+us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning used the outdated $120 under-65 grocery credit, yet it submitted $140, the old age-65+ amount, which does not apply to a 56-year-old. Under HB 231 the correct 2026 under-65 amount is $155."
+us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Wrongly treated the Idaho Grocery Credit as nonrefundable on the income tax return. The credit is refundable on the return and pays $155 regardless of tax liability, so state refundable credits are $155."
+us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Recognized the grocery credit but moved it out of refundable credits as a 'nonrefundable/other item.' The Idaho Grocery Credit is a refundable credit, worth $155 for this filer in 2026."
+us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,Concluded the grocery credit becomes nonrefundable and is absorbed whenever there is tax liability. The full Idaho Grocery Credit is refundable regardless of liability. It also used the stale $120 amount instead of the $155 in effect for 2026.
+us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Wrongly classed the Idaho Grocery Credit as nonrefundable against tax liability. It is a refundable credit, and this filer gets the full $155 for 2026."
+us,scenario_007,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,Correctly treated the Idaho Grocery Credit as refundable but used the pre-2025 amount of $120 per person. The amount increased to $155 starting in tax year 2025.
+us,scenario_007,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Invented a roughly $22,500 income phase-out for the Idaho Grocery Credit. The credit has no income phase-out and is reduced only for SNAP or incarceration months, so this filer gets the full $155."
+us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,Identified the refundable grocery credit but used the outdated $120 per-person amount instead of the $155 set by HB 231 for tax years 2025 and later.
+us,scenario_007,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used $100, the grocery credit amount from before 2023. The 2026 under-65 amount is $155."
+us,scenario_007,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Used the old $100 grocery credit amount instead of the 2026 amount of $155 per person for filers under 65.
+us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,Correctly treated the grocery credit as refundable but used the superseded $120 per-person amount. For 2026 it is $155.
+us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Said the household gets no refundable Idaho credits and missed the refundable Idaho Grocery Credit, which every full-year resident not receiving SNAP gets. For 2026 it is $155."
+us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly applied the under-65, no-SNAP grocery credit but used the old $120 amount. HB 231 raised it to $155 starting in 2025."
+us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,Used the pre-2025 grocery credit of $120 for filers under 65 instead of the $155 in effect for 2026.
+us,scenario_007,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Answered $0 and missed the refundable Idaho Grocery Credit, which is worth $155 for this full-year resident."
+us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,Correctly identified the refundable grocery credit but used the outdated $120 per-resident amount instead of the $155 that applies for 2026.
+us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,Applied the stale $120 grocery credit. The amount for filers under 65 has been $155 since tax year 2025.
+us,scenario_007,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Used the old $100 per-person grocery credit amount instead of the 2026 amount of $155 for filers under 65.
+us,scenario_007,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"Wrongly claimed Idaho's grocery credit was repealed. The credit is still in effect, refundable, and worth $155 per person in 2026, so the household's refundable credits are $155."
+us,scenario_007,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,Correctly found no income phase-out but used the outdated $100 grocery credit amount instead of $155 per person under 2026 law.
+us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,Found no refundable Idaho credit and missed the Idaho Grocery Credit. The credit is refundable and needs no special household facts beyond full-year residency without SNAP; it is $155 for 2026.
+us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Assumed refundable state credits need specific qualifying facts. The Idaho Grocery Credit only requires residency without SNAP receipt, so this filer gets $155."
+us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Applied an income phase-out that the Idaho Grocery Credit does not have. The credit is reduced only for SNAP or incarceration months, so the full $155 applies."
+us,scenario_007,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Said no refundable Idaho credits apply and missed the refundable Idaho Grocery Credit, which is $155 for this full-year resident in 2026."
+us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Used the old $100 grocery credit amount for filers under 65. After HB 231, the 2026 amount is $155."
+us,scenario_007,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,Correctly applied the refundable grocery credit with take-up but used the outdated $100 amount instead of $155.
+us,scenario_007,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,Correctly identified the grocery credit as the only refundable Idaho credit but used the superseded $120 amount instead of the 2026 amount of $155.
+us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Concluded no state refundable credits apply and missed the Idaho Grocery Credit, which is refundable and worth $155 for a full-year resident not receiving SNAP."
+us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so there was no answer to score against the $155 Idaho Grocery Credit."
+us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,Correctly applied the refundable grocery credit for one person under 65 but used the old $120 amount instead of the $155 in effect for 2026.
+us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Wrongly said Idaho has no state income tax. Idaho has an individual income tax, and its return carries the refundable Grocery Credit, which is $155 for this filer."
+us,scenario_007,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Guessed an inflation-adjusted amount of about $140, but the Idaho Grocery Credit is not indexed for inflation. The law sets the under-65 amount at $155 for 2025 and later."
+us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Said no meaningful refundable Idaho credits apply and missed the refundable Idaho Grocery Credit, which is $155 for this full-year resident."
+us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,Found no Idaho refundable credits and missed the Idaho Grocery Credit. The credit is refundable for any full-year resident not on SNAP and is $155 in 2026.
us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only NJ's upper CHIP ceiling (""well below NJ CHIP income limit (350% FPL)"") and never tested the lower Medicaid children's threshold that CHIP sits above. At ~$30,915 for a household of eight, the 17-year-old falls in the OLDER_CHILD Medicaid category, and Medicaid eligibility precludes CHIP, so the correct answer is 0."
us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model reasoned that ""very low income"" plus age under 19 establishes CHIP eligibility, inverting the actual rule: income far below the Medicaid children's limit routes the child to Medicaid, and CHIP covers only children who do not qualify for Medicaid. Child1 is Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is 0."
us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model built its entire test around the 350% FPL CHIP ceiling ($205,240 for a family of eight) and concluded eligibility because $30,915 fell under it, treating the absence of listed coverage as the only other condition. It omitted the Medicaid screen that runs first: a 17-year-old at roughly 54% FPL qualifies for NJ Medicaid in the OLDER_CHILD category, which by construction makes is_chip_eligible False."
@@ -509,18 +509,19 @@ us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disabil
us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"It assumed that turning 5 ages a child out of preschool Head Start. The program actually covers children from age 3 until the age of compulsory school attendance (6 in NJ), and PolicyEngine counts age 5 as eligible. Its answer ignores the household's SNAP-based categorical eligibility, which settles the income side."
us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"It placed age 5 outside the Head Start preschool age range. The rule actually runs from age 3 until the age of compulsory school attendance, so a 5-year-old is age-eligible. It never evaluated the SNAP categorical pathway or the income test, both of which qualify Child 5."
us,scenario_008,child5_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact instead of applying the age-based YOUNG_CHILD pathway. Child 5's age of 5 and MAGI of 0.55 times FPL satisfy that pathway, yielding eligibility."
-us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model wrote the correct rule — ""children under 5 years old are eligible"" — and then contradicted it, asserting ""age under 5"" for a child whose stated age is 5. WIC's child category (7 CFR 246.2) runs from the first birthday to the fifth birthday, so a child who has reached age 5 is out of category and no income test is ever reached; the model instead let the 185%-FPL income comparison and receipt of housing assistance carry the answer."
-us,scenario_008,child5_wic_eligible,claude-opus-5,llm_error,age_disability,False,"The model defined the WIC child category as ""1-4 completed years through age 5,"" stretching the upper bound a full year past the fifth birthday where the category actually terminates. It then compounded the error by invoking SNAP adjunctive eligibility, which only substitutes for the income test and cannot confer eligibility on someone who fails the categorical age requirement."
-us,scenario_008,child5_wic_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model stated the rule correctly (""a child under age 5 (up to 5th birthday)"") and then reversed itself, claiming eligibility ""extends through the child's 5th birthday, so a child who is currently 5 is still within the eligible age window."" Reaching age 5 ends the child category, and the adjunctive eligibility via housing assistance the model cited waives only the 185%-FPL income screen, not the age category."
-us,scenario_008,child5_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted ""Child 5 is age 5, eligible for WIC"" with no application of the category's upper age bound. WIC's child category ends at the fifth birthday, so age 5 is categorically ineligible; the model treated an age that terminates eligibility as though it established it."
-us,scenario_008,child5_wic_eligible,gemini-3.5-flash-lite,llm_error,age_disability,False,"The model checked only the income screen (""age 5 and eligible for WIC based on income"") and never applied the categorical age test. Income is not the discriminator here — the age-1 child qualifies on the identical household income — while age 5 places this child past the end of the WIC child category."
-us,scenario_008,child5_wic_eligible,glm-5.2,llm_error,age_disability,False,"The model stated ""WIC eligibility for children ends at age 5"" and then counted a child who is age 5 as satisfying the requirement, treating an exclusive upper bound as inclusive. Eligibility ends on the fifth birthday, so age 5 fails the category and the below-185%-FPL income finding the model relied on never comes into play."
-us,scenario_008,child5_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model asserted that age 5 is ""within the WIC child age range under the benchmark rules"" without checking the category's cutoff. The WIC child category spans ages 1 through 4 and terminates at the fifth birthday, so a child listed as 5 is outside it."
-us,scenario_008,child5_wic_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model placed age 5 inside ""the benchmark's WIC-eligible group"" on assertion alone, applying no upper age bound. The child category ends at the fifth birthday, which excludes this child; the same household income leaves the age-1 child eligible, confirming age is the binding test the model skipped."
-us,scenario_008,child5_wic_eligible,grok-4.3,llm_error,age_disability,False,"The model gave the bare claim ""Child 5 age 5 qualifies for WIC,"" performing no categorical age check. Age 5 is past the end of WIC's child category, which covers only children who have reached age 1 but not age 5."
-us,scenario_008,child5_wic_eligible,inkling,llm_error,age_disability,False,"The model conflated two mutually exclusive readings — ""the under-5/through-age-5 WIC range"" — and resolved the conflict in favor of the wrong one. The category is under 5: eligibility ends at the fifth birthday, so the child fails on age, and the below-185%-FPL income the model verified is irrelevant to that failure."
-us,scenario_008,child5_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model declared age 5 ""categorically eligible"" when the WIC child category in fact closes at the fifth birthday, then rested the answer on an income test that was never the binding constraint (its cited 185%-FPL figure of $45,080 corresponds to a two-person unit, not the eight-person household here). The age-1 child qualifies on the same $30,915 income, isolating age as the disqualifier."
-us,scenario_008,child5_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model paired an unexamined age claim with an income check (""age 5 and household income is within WIC income limits"") and never tested the upper bound of the child category. Eligibility as a child ends at the fifth birthday, so this child is out of category regardless of the income result."
+us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model correctly said 'children under 5 years old are eligible' but then counted a 5-year-old as under 5. WIC child eligibility ends at the 5th birthday, so Child 5 is categorically ineligible. The income and housing-assistance reasoning has no bearing on that."
+us,scenario_008,child5_wic_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model called a 5-year-old 'a child under 5', which contradicts the stated age. WIC's child category ends at the 5th birthday, so Child 5 is not eligible no matter how much the household earns."
+us,scenario_008,child5_wic_eligible,claude-opus-5,llm_error,age_disability,False,The model correctly said the WIC child category is '1-4 completed years' but then added 'through age 5'. A child with 5 completed years is outside the 1–4 range and has aged out of WIC. Adjunctive eligibility through SNAP only satisfies the income test and cannot make up for failing the age requirement.
+us,scenario_008,child5_wic_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model said WIC eligibility lasts until the 5th birthday, then treated a child who is already 5 as still eligible. A listed age of 5 means the 5th birthday has passed, so Child 5 fails the age test. Adjunctive income eligibility through housing assistance does not change that."
+us,scenario_008,child5_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model said a 5-year-old is WIC-eligible without checking the age cutoff. WIC covers children only until their 5th birthday (ages 1–4), so Child 5 is ineligible."
+us,scenario_008,child5_wic_eligible,gemini-3.5-flash-lite,llm_error,age_disability,False,"The model based eligibility on income alone and skipped the categorical age test. WIC serves children only until their 5th birthday, so a 5-year-old is excluded even though the household meets the income limit."
+us,scenario_008,child5_wic_eligible,glm-5.2,llm_error,age_disability,False,"The model correctly said 'WIC eligibility for children ends at age 5', then contradicted itself by treating a 5-year-old as meeting the age requirement. Eligibility ends at the 5th birthday, so Child 5 has aged out."
+us,scenario_008,child5_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model said age 5 falls within the WIC child age range. That range is ages 1–4 and ends at the 5th birthday, so Child 5 is categorically ineligible."
+us,scenario_008,child5_wic_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model put a 5-year-old in the WIC-eligible group. WIC's child category stops at the 5th birthday, so age 5 is outside it and Child 5 is ineligible."
+us,scenario_008,child5_wic_eligible,grok-4.3,llm_error,age_disability,False,"The model said age 5 qualifies for WIC without applying the age cutoff. WIC child eligibility ends at the 5th birthday, so a 5-year-old does not qualify."
+us,scenario_008,child5_wic_eligible,inkling,llm_error,age_disability,False,"The model merged 'under-5' and 'through-age-5' into one range, but these are different limits. WIC uses the under-5 limit (until the 5th birthday), so a 5-year-old is outside the child category even though household income is below 185% FPL."
+us,scenario_008,child5_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model's income test was right (the household is under 185% FPL for 8 people), but it called a 5-year-old categorically eligible. WIC's categorical child eligibility ends at the 5th birthday, so Child 5 fails the age test."
+us,scenario_008,child5_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model checked only the income limit and assumed age 5 meets WIC's age requirement. WIC covers children only until their 5th birthday, so Child 5 is ineligible."
us,scenario_008,child6_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model tested only NJ FamilyCare's upper CHIP ceiling (350% FPL) against household income and concluded the 1-year-old fits inside it, never applying the prior screen that CHIP covers only children who fail Medicaid eligibility. At roughly $30,915 for eight people the child clears New Jersey's young-child Medicaid category, and PolicyEngine's is_chip_eligible is False for any Medicaid-eligible child. Its income tally of $26,915 also dropped the $4,000 of self-employment income it had just listed."
us,scenario_008,child6_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model treated the 355% FPL CHIP ceiling as the only binding test and reasoned that being far below it makes the child qualify, when being far below it is precisely what places the child in the young-child Medicaid category. CHIP is residual coverage for children who do not qualify for Medicaid, so the child's Medicaid eligibility forces is_chip_eligible to False."
us,scenario_008,child6_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model stated the child ""qualifies for CHIP eligibility under income/age criteria,"" applying an age test and an upper income ceiling (over 300% FPL for infants in NJ) while omitting the not-Medicaid-eligible condition that CHIP requires. The 1-year-old is Medicaid-eligible under the YOUNG_CHILD category at this income, which makes CHIP eligibility False."
@@ -537,151 +538,153 @@ us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categori
us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 1 as within the preschool-age Head Start range. A one-year-old falls under Early Head Start rather than Head Start, and age alone does not establish income eligibility."
us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact and failed to apply the age-based YOUNG_CHILD pathway. Child 6 is age 1 and a dependent, and the household's MAGI of 0.55 times FPL satisfies the New Jersey income test for that category."
us,scenario_008,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It used a joint-filer EITC phase-out start of $30,050, which is below the household's AGI, and cut the $8,231 maximum to $8,173; in 2026 AGI of about $30,632 is still on the plateau. It set ACTC earned income at 0.9235 x SE income ($30,494) instead of SE income minus the half-SE-tax deduction ($30,517), and then submitted an unexplained $12,130, below its own component sum of $12,372."
-us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the EITC at about $1,395. A married couple with three or more children and about $30,600 of AGI is on the plateau and gets the full $8,231. It also dropped its own $4,245 ACTC figure to an unexplained $1,305 in the total; the correct ACTC is 15% x ($30,517 - $2,500) = $4,203."
-us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its ACTC of $4,202 was correct and its components summed to $12,469, close to the reference. It then applied an unsupported 'adjusting EITC' step that added about $700, which is impossible because the EITC was already at its maximum. It also put that maximum at $8,267 instead of the 2026 figure of $8,231."
-us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0. It also cut the ACTC to $388 on the theory that nonrefundable credits absorb it. Tax liability is zero, so the full earned-income-limited ACTC of 15% x ($30,517 - $2,500) = $4,203 is refundable."
-us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It put the EITC at about $7,700, but the household is below the 2026 joint phase-out start and gets the full $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting half the SE tax from SE income, and it added a $355 AOTC even though qualified education expenses are $0."
-us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used 2025 parameters: an $8,046 maximum and a phase-out start of $26,511, which is below the joint-filer threshold. That wrongly cut the EITC by $868 to $7,178, when the 2026 joint threshold is above its $30,632 AGI and the full $8,231 applies. Its ACTC also used 0.9235 x SE income ($4,199) instead of SE income minus half the SE tax ($4,203)."
-us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It applied an EITC phase-out from about $29,600 (giving $7,856) instead of the full $8,231. It treated the 15% x (earned income - $2,500) limit as if it were per child and claimed $8,500 of ACTC, when that limit caps the total ACTC at $4,203. It added a $1,000 AOTC with zero qualified expenses, then submitted an arbitrary 'reconciled' $12,900."
-us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It put the EITC at $8,021 instead of the full 2026 maximum of $8,231; the household's AGI is below the joint phase-out start. It computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction, which gives $30,517 and $4,203."
-us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the $3,000 ACTC earned-income threshold instead of the 2026 threshold of $2,500, on gross earnings of $30,800 ($4,170 instead of $4,203). It also put the EITC at $7,787 instead of the full $8,231 plateau amount."
-us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $4,124 matches 15% x ($30,494 - $3,000). That uses the $3,000 threshold instead of $2,500, and 0.9235 x SE income instead of SE income minus half the SE tax; the correct ACTC is $4,203. Its EITC of $8,190 is also below the 2026 three-child maximum of $8,231."
-us,scenario_008,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,247 instead of $8,231. It measured ACTC earned income as 0.9235 x SE income ($30,494), when the correct figure is $4,000 - $282.59 of half SE tax, giving $30,517.41 and an ACTC of $4,202.61 rather than $4,199.10."
-us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It cut the EITC to $7,691 even though the household's AGI of about $30,632 is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,199 used 0.9235 x SE income instead of SE income minus half the SE tax, which gives $4,203."
-us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no breakdown; its round $11,400 is $1,034 short of the correct total. That total is the $8,231 maximum EITC plus ACTC of 15% x ($30,517 - $2,500) = $4,203. Its claim of the 'full Child Tax Credit' is wrong too: the refundable portion is limited by the 15% earned-income formula, not paid at the per-child amount, and its total is consistent with neither."
-us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its ACTC of $4,124 equals 15% x ($30,494 - $3,000). That uses the $3,000 threshold instead of $2,500, and 0.9235 x SE income instead of SE income minus half the SE tax; the correct ACTC is $4,203. Its EITC of about $8,200 is also below the 2026 maximum of $8,231."
-us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It phased the EITC down to $7,128 even though AGI is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,128 matches 15% x ($30,517 - $3,000): it used the $3,000 threshold instead of $2,500, which gives $4,203."
-us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no component breakdown, and its $9,750 is $2,684 below the correct total of the $8,231 maximum EITC plus a $4,203 ACTC (15% x ($30,517 - $2,500)). The figure implies it sharply understated the EITC, the ACTC, or both, rather than computing either from the plateau and the earned-income formula."
-us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,203 was correct. For the EITC it used the 2025 three-child maximum of $8,046 instead of the 2026 maximum of $8,231, which understated the total by $185."
-us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It gave no component breakdown, and its $11,940 is $494 below the correct total. That total is the full 2026 three-child EITC of $8,231 (AGI is below the joint phase-out start) plus ACTC of 15% x ($30,517 - $2,500) = $4,203. The shortfall is consistent with either a stale or phased-out EITC or the $3,000 ACTC threshold."
-us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,128 matches 15% x ($30,517 - $3,000), so it used the $3,000 earned-income threshold instead of $2,500, which gives $4,203. Its EITC of $8,233 is essentially the $8,231 maximum."
-us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive calculation to evaluate. The correct total is the $8,231 EITC plus a $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,360 instead of $8,231. For ACTC earned income it used 0.9235 x SE income ($30,494) instead of SE income minus the half-SE-tax deduction ($30,517), giving $4,199 instead of $4,203."
-us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It denied both credits, claiming the qualifying-child facts were not established. The six resident children aged 1-17 are all EITC qualifying children, and the five under 17 are CTC qualifying children. With $30,517 of earned income, that yields an $8,231 EITC and a $4,203 refundable CTC."
-us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0, claiming no qualifying income structure. The household has $30,517 of earned income and six qualifying children, which yields the full $8,231 three-child EITC plus a refundable CTC of 15% x ($30,517 - $2,500) = $4,203."
-us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct. It computed ACTC on gross earnings of $30,800, but earned income counts SE income net of the half-SE-tax deduction ($4,000 - $282.59), which gives $30,517.41. The ACTC is therefore $4,202.61, not $4,245."
-us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the 2025 three-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It also computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction ($30,517, giving $4,203)."
-us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its ACTC of $4,245 is 15% of gross earnings of $30,800 minus $2,500. It never subtracted the $282.59 half-SE-tax deduction from SE income; the correct earned income is $30,517.41 and the correct ACTC is $4,202.61."
-us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 equals the correct $8,231 EITC plus an ACTC of $4,245, which is 15% x ($30,800 - $2,500) on gross earnings. It did not reduce SE income by the half-SE-tax deduction; the correct earned income of $30,517.41 gives an ACTC of $4,202.61."
-us,scenario_008,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It put the EITC at $8,162 instead of the full 2026 maximum of $8,231; AGI is below the joint phase-out start. It computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction, which gives $4,203."
-us,scenario_008,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $4,203 was correct. It overstated the 2026 three-child EITC maximum as $8,342 instead of $8,231."
-us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"It treated zero net tax as a reason for $0 refundable credits, which gets refundability backwards: the EITC and the refundable CTC are paid even when there is no liability. This household gets the $8,231 EITC plus a $4,203 refundable CTC."
-us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules to the CTC: a $3,000 ACTC earned-income threshold and a $1,000 per-child amount. In 2026 the threshold is $2,500, which gives an ACTC of $4,203 instead of $4,124. It also overstated the EITC maximum as $8,247 instead of $8,231."
-us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed ACTC with the $3,000 earned-income threshold ($4,124) instead of the 2026 threshold of $2,500, which gives 15% x ($30,517 - $2,500) = $4,203. It also rounded the EITC maximum to $8,250 instead of $8,231."
-us,scenario_008,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its EITC of $8,231 was correct. For ACTC it used the 'post-sunset' $3,000 threshold instead of the 2026 threshold of $2,500, and 0.9235 x SE income ($30,494) instead of SE income minus half the SE tax ($30,517); the ACTC is $4,202.61, not $4,124.10."
-us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"It phased the EITC down to $7,901 even though AGI of $30,632 is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,199 used 0.9235 x SE income instead of SE income minus half the SE tax, which gives $4,203."
-us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting the half-SE-tax deduction; the correct earned income of $30,517 gives $4,203."
-us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive calculation to evaluate. The correct total is the $8,231 EITC plus a $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 and its AOTC of $0 were correct. Its ACTC of $4,245 applies 15% to gross earnings of $30,800 minus $2,500, but earned income counts SE income net of the $282.59 half-SE-tax deduction. That makes earned income $30,517.41 and the ACTC $4,202.61."
-us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used $7,152, the two-child EITC maximum, instead of the $8,231 maximum for three or more children. It then applied a phase-out from $28,120, which is below the 2026 joint threshold, reaching $6,588. Its ACTC used gross earnings of $30,800 ($4,245) instead of SE income minus half the SE tax ($4,203)."
-us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,262 instead of $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting the half-SE-tax deduction from SE income; the correct earned income of $30,517 gives $4,203."
-us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It used the 2025 EITC maximum of $8,046 and a phase-out start of $26,210, which is below the joint-filer threshold, and cut the EITC to $7,144 instead of the full $8,231. It also added a $1,000 refundable AOTC even though qualified education expenses are $0, and its ACTC used 0.9235 x SE income ($4,199 instead of $4,203)."
-us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It limited the ACTC to $1,600 based on 'remaining tax liability room'. The ACTC is limited by 15% of earned income over $2,500 ($4,203), not by tax liability. It also understated the EITC at $6,801 instead of the $8,231 plateau amount, and it added $6,801 + $1,600 incorrectly to get $6,586."
+us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It applied an EITC phase-out starting near $30,050, but the 2026 MFJ threshold is above AGI, so the full $8,231 applies. It also computed the ACTC from earned income of $30,494 (0.9235×SE) instead of $30,517.41 (SE income minus half of SE tax). Finally, it submitted $12,130, which is below even its own $12,372 sum."
+us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It put the EITC at about $1,395. A married couple with three or more children and about $30.5k of earned income is on the plateau and gets the full $8,231. It also computed an ACTC of $4,245 but then used an unexplained $1,305; the correct figure is $4,202.61."
+us,scenario_008,federal_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It acknowledged the EITC was positive and still reported $0. That drops the $8,231 maximum EITC and the $4,202.61 refundable CTC (15% of the $28,017.41 of earnings above $2,500), both of which are paid regardless of tax liability."
+us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its ACTC of $4,202 was right, but it overstated the 2026 EITC maximum as $8,267 instead of $8,231. It then inflated its own $12,469 total to an unsupported $13,167, even though no phase-out or other credit applies."
+us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0, so the AOTC is $0. It also cut the ACTC to about $388 for supposed nonrefundable use. With zero tax liability, the full $4,202.61 (15% of earned income above $2,500) is refundable."
+us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It reduced the EITC to about $7,700, but AGI is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. It also used gross $30,800 earnings for the ACTC instead of $30,517.41 after the half-SE-tax deduction, and it added a $355 AOTC even though there are no qualified expenses."
+us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 2025 EITC parameters with a $26,511 MFJ phase-out start, which reduced an $8,046 maximum to $7,178. The 2026 maximum is $8,231, and the 2026 MFJ phase-out threshold is above AGI, so there is no reduction. It also computed the ACTC on $30,494 of earned income instead of $30,517.41."
+us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It phased the EITC down to $7,856 from a ~$29,600 threshold instead of using the unreduced 2026 maximum of $8,231. It treated the ACTC as up to $8,500 even though the 15% earned-income formula limits it to $4,202.61. It also added a $1,000 AOTC with $0 qualified expenses and settled on an unsupported $12,900."
+us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated the EITC at $8,021 instead of the full $8,231 2026 maximum. It also computed the ACTC on gross $30,800 earnings. Earned income must reflect the half-SE-tax deduction ($30,517.41), which gives $4,202.61 rather than $4,245."
+us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It phased the EITC down to $7,787 instead of using the full $8,231 maximum, since AGI is below the 2026 MFJ threshold. It also used the expired pre-TCJA $3,000 ACTC earnings threshold instead of the permanent $2,500 threshold."
+us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $4,124 matches 15% × ($30,494 − $3,000), which uses the expired $3,000 threshold instead of $2,500 and omits the half-SE-tax adjustment to earned income. Its EITC of $8,190 is also below the $8,231 2026 maximum."
+us,scenario_008,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,247 instead of $8,231. It also counted SE earned income as 0.9235×$4,000 = $3,694 instead of $4,000 minus half of SE tax ($3,717.41). That gives an ACTC of $4,199.10 instead of $4,202.61."
+us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It phased the EITC down to $7,691, but AGI is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. Its ACTC used $30,494 of earned income instead of $30,517.41."
+us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said the household gets the maximum EITC plus the CTC but reported $11,400, which is inconsistent with that. The maximum EITC is $8,231 and the refundable CTC is $4,202.61, which together give $12,433.61."
+us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its ACTC of $4,124 reflects the expired $3,000 earnings threshold applied to $30,494. The correct figure is 15% × ($30,517.41 − $2,500) = $4,202.61. It also rounded the EITC to $8,200 instead of the $8,231 maximum."
+us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a phase-out that cut the EITC to $7,128, but AGI is below the 2026 MFJ threshold, so the full $8,231 applies. It also computed the ACTC as $4,128 using the expired $3,000 threshold instead of $2,500."
+us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no breakdown, and $9,750 is about $2,700 short. The correct total is the full $8,231 2026 EITC plus the $4,202.61 refundable CTC, so the answer understates one or both credits."
+us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,203 was correct, but it used the 2025 three-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231, which is exactly the $185 gap."
+us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no component breakdown, and $11,940 is about $494 short. That shortfall implies a phased-down or outdated EITC instead of the full $8,231 2026 maximum, added to the $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,128 uses the expired $3,000 earnings threshold instead of $2,500. The correct figure is 15% × ($30,517.41 − $2,500) = $4,202.61. Its EITC of $8,233 is also $2 above the $8,231 maximum."
+us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no answer or explanation for federal_refundable_credits, so no value could be scored."
+us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,360 instead of $8,231. It also computed the ACTC from $30,494 of earned income (0.9235×SE) instead of $30,517.41 (SE income minus half of SE tax)."
+us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It wrongly concluded that the qualifying children were not established. Six resident children aged 1–17 qualify for the three-or-more-child EITC ($8,231), and the five under 17 generate a refundable CTC of $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It reported $0 even though the household has $30,517.41 of earned income and six qualifying children. That income produces the full $8,231 EITC and a refundable CTC of $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but it computed the ACTC on gross $30,800 of earnings. Self-employment earned income must be reduced by the half-SE-tax deduction ($282.59), which makes earned income $30,517.41 and the ACTC $4,202.61 rather than $4,245."
+us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction, which gives $4,245 instead of $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its ACTC of $4,245 treats the full $4,000 of SE income plus wages as earned income. Earned income nets out half of SE tax, giving $30,517.41 and an ACTC of $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 equals $8,231 + $4,245, which means it computed the refundable CTC on gross $30,800 of earnings. Subtracting the $282.59 half-SE-tax deduction gives earned income of $30,517.41 and a refundable CTC of $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It understated the EITC at $8,162 instead of the full $8,231 2026 maximum, since AGI is below the phase-out threshold. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction."
+us,scenario_008,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $4,203 was correct, but it overstated the 2026 three-child EITC maximum as $8,342 instead of $8,231."
+us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"It treated refundable credits as inapplicable because tax liability was already zero. Refundable credits are paid out regardless of liability, so the $8,231 EITC and the $4,202.61 refundable CTC are both payable in full."
+us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules: a $3,000 ACTC earnings threshold and a $1,000-per-child credit. The permanent 2026 law uses a $2,500 threshold and a $1,700 refundable cap, which gives an ACTC of $4,202.61. It also overstated the EITC maximum as $8,247 instead of $8,231."
+us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the expired $3,000 ACTC earnings threshold instead of the permanent $2,500 threshold. That gave $4,124 instead of $4,202.61. It also rounded the EITC to $8,250 instead of $8,231."
+us,scenario_008,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its EITC of $8,231 was correct, but it assumed the TCJA had sunset and used a $3,000 ACTC threshold on $30,494 of earned income. Permanent law keeps the $2,500 threshold, and earned income is $30,517.41, so the ACTC is $4,202.61."
+us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It phased the EITC down to $7,901, but AGI of about $30,632 is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. Its ACTC used $30,494 of earned income instead of $30,517.41."
+us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 EITC maximum as $8,271 instead of $8,231. It also computed the ACTC on gross $30,800 of earnings instead of $30,517.41 after the half-SE-tax deduction."
+us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer or explanation for federal_refundable_credits, so no value could be scored."
+us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its refundable CTC of $4,245 uses gross $30,800 of earnings. Earned income must subtract half of SE tax ($282.59), which gives 15% × $28,017.41 = $4,202.61."
+us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a three-child EITC maximum of about $7,152 (closer to the two-child maximum) and phased it out from a $28,120 threshold. The 2026 three-child maximum is $8,231, and it is unreduced at this AGI. Its ACTC also used gross $30,800 instead of $30,517.41."
+us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It overstated the 2026 EITC maximum as $8,262 instead of $8,231. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction."
+us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0, so the AOTC is $0. It also used a 2025-level $8,046 EITC maximum phased out from a $26,210 threshold, which gave $7,144. The full 2026 maximum of $8,231 applies because AGI is below the MFJ threshold."
+us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It wrongly limited the ACTC to $1,600 of 'tax liability room'. The ACTC is 15% of earned income above $2,500 ($4,202.61) and does not depend on liability. It also understated the EITC at $6,801 instead of $8,231, and its total of $6,586 does not equal its own components."
us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required a separately provided school-meals eligibility indicator and treated housing assistance as the only relevant benefit fact. It failed to compute the income-based free tier at 55% of the federal poverty guideline and also missed categorical eligibility through the household's computed SNAP eligibility.
-us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted income exceeds New Jersey's guidelines, but it never scaled the FPL to the eight-person household: 138% of the eight-person 2026 FPL is roughly $76,000 while total countable MAGI is $30,915, i.e. 0.55 x FPL. Its denial is consistent with testing the household's combined $30,915 against a one- or two-person 138% FPL limit (roughly $21,600 / $29,200), which is the only way this income clears any NJ adult Medicaid threshold."
-us,scenario_008,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model correctly identified that NJ adult eligibility runs through the ACA expansion rule but applied the 138% FPL limit without sizing it to the eight-person household, so $30,915 of MAGI read as too high when it is 0.55 x FPL against a 138% limit of roughly $76,000. Household size drives the FPL denominator in the MAGI test, and with six children plus two adults this family sits at less than half the expansion ceiling."
-us,scenario_008,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicit eligibility indicator in the input facts and defaulted to 0 when none appeared, instead of computing it from the facts given. The determination is fully specified by the supplied data: age 42 (under 65), New Jersey (an ACA expansion state), and household MAGI of $30,915 for eight people = 0.55 x FPL, which places the head squarely in the MAGI-based ADULT expansion category."
-us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the pre-ACA categorical framework, in which a non-disabled, non-elderly adult needed to be a pregnant woman, a caretaker relative under a low state limit, or aged/blind/disabled to qualify. New Jersey's ACA expansion covers non-disabled adults 19-64 up to 138% FPL on MAGI alone, and at 0.55 x FPL the head qualifies under that ADULT category; the head is also a parent of six resident children, so even the caretaker-relative pathway would not have supported a denial."
-us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model computed the MAGI test correctly (income under the 138% FPL line for eight people, head in an eligible non-elderly adult category) and then overrode its own result with a fabricated rule that receiving housing assistance disqualifies adults from Medicaid. MAGI Medicaid has no housing-assistance bar and no asset test; Section 8 or public housing subsidies are excluded from MAGI income entirely, so the household's receipt of housing assistance leaves the ADULT expansion determination untouched."
+us,scenario_008,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated the head as ineligible because they are a non-disabled adult. It ignored NJ's ACA expansion pathway, which covers any adult under 65 with MAGI at or below 138% FPL. The head's 8-person household MAGI is 0.55× FPL, so they qualify in the ADULT category."
+us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income exceeds NJ guidelines, but it never compared household MAGI (about $30.8k) with 138% FPL for an 8-person household (about $77k). The household is at 0.55× FPL, far below the expansion limit, so the head is eligible."
+us,scenario_008,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model concluded the head fails NJ's adult rules ""at this income level."" It did not scale the FPL to the 8-person household. Against that FPL, the roughly $30.8k MAGI is 0.55× FPL, well under the 138% ACA adult expansion limit."
+us,scenario_008,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to 0 because no explicit Medicaid eligibility indicator was listed. Eligibility has to be derived from the facts given: the head is under 65, lives in an expansion state (NJ), and has household MAGI of 0.55× FPL. Those facts place the head in the ACA adult expansion category, which is eligible."
+us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model treated non-disability as disqualifying. It ignored the ACA adult expansion category, which in NJ covers non-disabled adults under 65 with MAGI at or below 138% FPL. The head's MAGI is 0.55× FPL, so they are eligible."
+us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly found the head in the expansion category with income below 138% FPL. It then applied a made-up rule that receiving housing assistance disqualifies adults from Medicaid. No such exclusion exists, so the head's ACA adult expansion eligibility at 0.55× FPL stands."
us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly transferred the infant's categorical eligibility to the Head through household membership. WIC requires the applicant personally to be a pregnant, postpartum, or breastfeeding woman, infant, or child under age 5; the 42-year-old Head meets none of those categories and is therefore ineligible despite household income below the limit."
us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,payroll_tax,claude-fable-5,reference_engine_defect,payroll_tax_base,False,"Added the NJ UI (0.3825%) and workforce/SWF (0.0425%) employee contributions, which New Jersey requires and the frozen reference leaves out. It also set the TDI employee rate to 0% instead of 0.19% ($50.92) and used a stale 0.09% FLI rate instead of 0.23% ($61.64), giving $128.65 of NJ tax, against the frozen reference's $112.56 and the $226.46 NJ component of the exclusion's corrected value."
-us,scenario_008,payroll_tax,claude-fable-5.1,reference_engine_defect,payroll_tax_base,False,"Stopped at federal FICA ($2,050.20) and left out NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions, which add $112.56."
-us,scenario_008,payroll_tax,claude-haiku-4.5,reference_engine_defect,payroll_tax_base,False,"Computed FICA of $2,050.20 correctly, then dropped its own NJ calculation and added an invented '$2.12 minimum assessment'. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), which is $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46."
-us,scenario_008,payroll_tax,claude-opus-4.7,reference_engine_defect,payroll_tax_base,False,"Treated the 2026 NJ TDI employee rate as 0%, was unsure of the FLI rate, and fell back to FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64)."
-us,scenario_008,payroll_tax,claude-opus-4.8,reference_engine_defect,payroll_tax_base,False,Assumed NJ employee UI/SDI/FLI contributions are 'not modeled' and set them to $0. The output counts NJ TDI ($50.92 at 0.19%) and FLI ($61.64 at 0.23%) as mandatory employee state payroll taxes.
-us,scenario_008,payroll_tax,claude-opus-5,reference_engine_defect,payroll_tax_base,False,"Explicitly left NJ employee TDI and FLI out of the measure, even though the question's definition includes mandatory employee state payroll taxes. That drops $112.56 (0.19% TDI + 0.23% FLI), and the model also rounded FICA down to $2,050."
-us,scenario_008,payroll_tax,claude-opus-5.5,reference_engine_defect,payroll_tax_base,False,"Applied only the 7.65% federal FICA rate and never added NJ's employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions."
-us,scenario_008,payroll_tax,claude-sonnet-4.6,reference_engine_defect,payroll_tax_base,False,"Wrongly concluded that NJ TDI is employer-funded with a 0% employee rate and that FLI does not count as a payroll tax, so it reported FICA only. NJ withholds TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) from employees, and both are included."
-us,scenario_008,payroll_tax,claude-sonnet-5,reference_engine_defect,payroll_tax_base,False,"Recognized that NJ has mandatory employee UI/DI/FLI taxes and estimated about $201, then set them to $0 because of 'cap uncertainty'. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46. Wages are well below the cap."
-us,scenario_008,payroll_tax,claude-sonnet-5.5,reference_engine_defect,payroll_tax_base,False,Assumed NJ employee payroll taxes are 'not modeled' and reported FICA only. That omits TDI of $50.92 (0.19%) and FLI of $61.64 (0.23%).
-us,scenario_008,payroll_tax,deepseek-v4-flash-0731,reference_engine_defect,payroll_tax_base,False,"Computed FICA only and rounded $2,050.20 down to $2,050. It omitted the NJ employee TDI (0.19%) and FLI (0.23%) contributions totaling $112.56."
-us,scenario_008,payroll_tax,deepseek-v4-pro,reference_engine_defect,payroll_tax_base,False,"Included NJ TDI and FLI but used outdated rates: 0.10% TDI ($26.80) and 0.09% FLI ($24.12). The 2026 rates are 0.19% TDI ($50.92) and 0.23% FLI ($61.64), so NJ tax was understated by $61.64."
-us,scenario_008,payroll_tax,deepseek-v4-pro-0813,reference_engine_defect,payroll_tax_base,False,"Put NJ employee payroll taxes at $154.10 (about 0.575% of wages). That is $41.54 above the frozen reference's TDI at 0.19% plus FLI at 0.23% ($112.56) and $72.36 below the $226.46 of the exclusion's corrected value, which adds the worker unemployment and workforce contributions New Jersey requires and the frozen reference leaves out."
-us,scenario_008,payroll_tax,deepseek-v4.1-flash,reference_engine_defect,payroll_tax_base,False,"Reported federal FICA only and omitted NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions."
-us,scenario_008,payroll_tax,gemini-3-flash-preview,reference_engine_defect,payroll_tax_base,False,"Added 0.515% for NJ SUI plus FLI ($138.02). It counted the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and left out TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44."
-us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA and rounded to $2,050, omitting the NJ employee TDI and FLI contributions of $112.56."
-us,scenario_008,payroll_tax,gemini-3.1-pro-preview,reference_engine_defect,payroll_tax_base,False,"Added about $138 of NJ SUI/FLI/TDI. It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and misstated the rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, and the model's $2,188 is $88.66 below the corrected value, $2,276.66."
-us,scenario_008,payroll_tax,gemini-3.5-flash,reference_engine_defect,payroll_tax_base,False,"Added $123.80 of NJ UI/FLI. It counted the UI employee contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI (0.19%, $50.92). Its $2,174 is $102.66 below the exclusion's corrected value, $2,276.66."
-us,scenario_008,payroll_tax,gemini-3.5-flash-lite,reference_engine_defect,payroll_tax_base,False,Applied only 7.65% federal FICA and never added NJ's employee TDI (0.19%) and FLI (0.23%) contributions of $112.56.
-us,scenario_008,payroll_tax,gemini-3.6-flash,reference_engine_defect,payroll_tax_base,False,"Added $138.02 of NJ SUI/FLI. It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44."
-us,scenario_008,payroll_tax,gemini-3.7-flash,reference_engine_defect,payroll_tax_base,False,"Added $139.32 of NJ FLI/SUI/WF. It included the UI and workforce development contributions, which New Jersey requires and the frozen reference leaves out, and left out TDI (0.19%, $50.92). Its $2,189.52 is $87.14 below the exclusion's corrected value, $2,276.66."
-us,scenario_008,payroll_tax,gemini-3.8-flash,reference_engine_defect,payroll_tax_base,False,Reported FICA only and omitted the NJ employee TDI ($50.92) and FLI ($61.64) contributions.
-us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_008,payroll_tax,glm-5.3,reference_engine_defect,payroll_tax_base,False,Explicitly treated NJ employee state payroll tax as 'not modeled' and reported FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64).
-us,scenario_008,payroll_tax,gpt-5.4-mini,reference_engine_defect,payroll_tax_base,False,"Correctly worked out $1,661.60 Social Security and $388.60 Medicare, then submitted an unsupported $4,097, roughly double its own FICA sum. It also added no NJ state payroll tax; the frozen reference's total is $2,050.20 + $112.56 NJ TDI/FLI ($2,162.76), and the exclusion's corrected value adds $113.90 of NJ worker unemployment and workforce contributions, for $2,276.66."
-us,scenario_008,payroll_tax,gpt-5.4-nano,reference_engine_defect,payroll_tax_base,False,"Reported $1,525, which is below even the 7.65% FICA on $26,800 ($2,050.20), so its rate arithmetic was wrong. It also added nothing for NJ TDI and FLI ($112.56)."
-us,scenario_008,payroll_tax,gpt-5.5,reference_engine_defect,payroll_tax_base,False,"Applied a combined 0.9425% NJ rate ($252.59) that includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, along with an overstated FLI rate. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $252.59 exceeds by $26.13."
-us,scenario_008,payroll_tax,gpt-5.6-luna,reference_engine_defect,payroll_tax_base,False,"Added $253.26 of NJ contributions (0.945%). That is 0.425% UI/WF/SWF, which New Jersey requires and the frozen reference leaves out, plus TDI 0.19% and FLI at 0.33% instead of the 2026 rate of 0.23%, so it exceeds the $226.46 of the exclusion's corrected value by $26.80."
-us,scenario_008,payroll_tax,gpt-5.6-sol,reference_engine_defect,payroll_tax_base,False,"Added $263.98 of NJ contributions (0.985%). That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, and overstates FLI at 0.33% and TDI at 0.23%. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI, or $112.56; the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52."
-us,scenario_008,payroll_tax,gpt-5.6-terra,reference_engine_defect,payroll_tax_base,False,"Added $202.34 of NJ contributions (0.755%). That includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, and misstates TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $202.34 falls $24.12 short of."
-us,scenario_008,payroll_tax,gpt-6-astra,reference_engine_defect,payroll_tax_base,False,"Used the correct TDI (0.19%) and FLI (0.23%) rates and added the 0.425% UI/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. The frozen reference counts only TDI and FLI ($112.56); the model's $226.46 NJ addition, for $2,276.66, is the exclusion's corrected value."
-us,scenario_008,payroll_tax,gpt-6-luna,reference_engine_defect,payroll_tax_base,False,"Added $263.98 of NJ contributions (0.985%). That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, with overstated TDI/FLI rates. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI ($112.56); the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52."
-us,scenario_008,payroll_tax,gpt-6-sol,reference_engine_defect,payroll_tax_base,False,"Applied TDI at 0.19% and FLI at 0.23% correctly and added the 0.425% unemployment/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. Its NJ addition of $226.46, for $2,276.66, is the exclusion's corrected value; the frozen reference's is $112.56."
-us,scenario_008,payroll_tax,gpt-6.1-sol,reference_engine_defect,payroll_tax_base,False,"Added $215.07 of NJ contributions: 0.3825% UI/workforce ($102.51) plus TDI 0.19% and FLI 0.23%. That 0.3825% is the worker unemployment contribution, which New Jersey requires and the frozen reference leaves out; the model did not add the 0.0425% workforce share ($11.39), so its $2,265.27 is $102.51 above the frozen reference's $2,162.76 and $11.39 below the exclusion's corrected value, $2,276.66."
-us,scenario_008,payroll_tax,grok-4.3,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA (rounded to $2,050) and omitted NJ's mandatory employee TDI (0.19%) and FLI (0.23%) contributions of $112.56."
-us,scenario_008,payroll_tax,grok-4.5,reference_engine_defect,payroll_tax_base,False,"Included NJ TDI and FLI but at a stale combined rate of 0.18% ($48.24). The 2026 rates are 0.19% TDI plus 0.23% FLI, or 0.42% ($112.56)."
-us,scenario_008,payroll_tax,grok-4.6,reference_engine_defect,payroll_tax_base,False,"Added the NJ employee UI contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, plus about $75 of TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds UI and the 0.0425% workforce share ($113.90), for $226.46, and the model's $2,228 is $48.66 below the corrected value, $2,276.66."
-us,scenario_008,payroll_tax,grok-4.7,reference_engine_defect,payroll_tax_base,False,"Included a 0.425% UI/workforce contribution, which New Jersey requires and the frozen reference leaves out, and overstated TDI (0.23% vs 0.19%) and FLI (0.33% vs 0.23%). That gives $263.98 of NJ tax, $151.42 above the frozen reference's $112.56 and $37.52 above the $226.46 of the exclusion's corrected value."
-us,scenario_008,payroll_tax,grok-build-0.1,reference_engine_defect,payroll_tax_base,False,"Explicitly assumed there are no state payroll taxes and reported FICA only (rounded to $2,050). It omitted NJ TDI ($50.92) and FLI ($61.64)."
-us,scenario_008,payroll_tax,inkling,reference_engine_defect,payroll_tax_base,False,"Added the NJ UI employee contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, used a stale 0.09% FLI rate instead of 0.23%, and omitted TDI at 0.19% and the 0.0425% workforce share. That gave about $127 of NJ tax, against the frozen reference's $112.56 and the $226.46 of the exclusion's corrected value; its $2,177 is $99.66 below the corrected value, $2,276.66."
-us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_008,payroll_tax,kimi-k3,reference_engine_defect,payroll_tax_base,False,"Added $257.95 of NJ contributions, including the UI/workforce employee contribution, which New Jersey requires and the frozen reference leaves out, and inflated TDI/FLI rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $257.95 exceeds by $31.49."
-us,scenario_008,payroll_tax,minimax-m3,reference_engine_defect,payroll_tax_base,False,"Computed FICA only (rounded to $2,050) and never added NJ's mandatory employee TDI (0.19%) and FLI (0.23%) contributions of $112.56."
-us,scenario_008,payroll_tax,ox-alpha,reference_engine_defect,payroll_tax_base,False,Assumed no NJ employee payroll tax is modeled and reported FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64).
-us,scenario_008,payroll_tax,qwen-3.7-max,reference_engine_defect,payroll_tax_base,False,"Used 2024-era assumptions: NJ employee DI and FLI both at 0%, plus the 0.3825% UI/WF/SWF contribution ($102.51). For 2026 the frozen reference counts TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) and leaves out UI, which New Jersey requires; the exclusion's corrected value counts UI at 0.3825% and the workforce funds at 0.0425% ($113.90), for $2,276.66, which its $2,152.71 misses by $123.95: TDI, FLI and the workforce share ($11.39)."
-us,scenario_008,payroll_tax,qwen3.8-max,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA to the spouse's wages and omitted NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions."
+us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used 2024 NJ rates: TDI at 0% instead of 0.19% ($50.92) and FLI at 0.09% ($24.12) instead of 0.23% ($61.64). It also summed its own NJ items ($102.51 + $24.12 + $11.39 = $138.02) as $128.65, so its NJ total came out $97.81 short of $226.46."
+us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($2,050.20) and left out the $226.46 of mandatory NJ employee contributions: TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425% on $26,800. The question explicitly lists these as part of payroll_tax."
+us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Got FICA right ($2,050.20), then dropped its own NJ UI estimate and replaced it with a made-up $2.12 'minimum assessment'. The correct NJ employee contributions total $226.46: TDI $50.92, FLI $61.64, UI $102.51 and workforce $11.39."
+us,scenario_008,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Stopped at federal FICA of $2,050.20 and never added NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) or workforce (0.0425%) contributions. Those total $226.46 on $26,800."
+us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Recognized that NJ has employee UI, SDI and FLI contributions, but left all of them out because it was unsure of the rates. It wrongly took the 2026 TDI rate to be 0%, when the correct 2026 rates are TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%, which add $226.46."
+us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Assumed NJ's mandatory employee UI, SDI and FLI contributions were 'not modeled' and set them to $0. They are part of payroll_tax and come to $226.46 on $26,800 (TDI $50.92, FLI $61.64, UI $102.51, workforce $11.39)."
+us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Explicitly excluded NJ employee UI, TDI and FLI from the measure, even though the question defines payroll_tax to include mandatory employee state payroll taxes. That left out $226.46, and it also rounded FICA down to $2,050."
+us,scenario_008,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only the 7.65% federal employee FICA rate. It left out NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions, which total $226.46."
+us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Wrongly concluded that NJ TDI is employer-funded and that NJ employee contributions fall outside 'mandatory employee state payroll taxes', so it used FICA alone. NJ withholds employee TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%, which add $226.46."
+us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Estimated about $201 of NJ UI/DI/FLI contributions, then dropped them from the total, citing wage-cap uncertainty. $26,800 is below every NJ wage base, so the full $226.46 applies: TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%."
+us,scenario_008,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed NJ employee payroll taxes are 'not modeled' and counted only FICA. That left out $226.46 of mandatory NJ employee TDI, FLI, UI and workforce contributions on $26,800."
+us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Counted only federal Social Security and Medicare, then rounded to $2,050. It never added the $226.46 of mandatory NJ employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions."
+us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Included only NJ TDI at 0.10% ($26.80) and FLI at 0.09% ($24.12). The 2026 rates are 0.19% ($50.92) and 0.23% ($61.64), and it left out the employee UI (0.3825%, $102.51) and workforce (0.0425%, $11.39) contributions entirely."
+us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Put NJ employee contributions at $154.10, a combined 0.575% of wages. The correct combined rate is 0.845% ($226.46): TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%. That leaves the NJ component $72.36 short."
+us,scenario_008,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Computed only employee FICA of $2,050.20. It left out the mandatory NJ employee TDI, FLI, UI and workforce contributions, which add $226.46 on $26,800."
+us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Applied a combined 0.515% NJ rate ($138.02), which is UI/workforce 0.425% plus the 2024 FLI rate of 0.09%. It left out the 0.19% employee TDI and used 0.09% instead of the 2026 FLI rate of 0.23%, so NJ came out $88.44 short."
+us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Applied only the 7.65% federal FICA rate and rounded to $2,050. It left out the $226.46 of mandatory NJ employee TDI, FLI, UI and workforce contributions."
+us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Estimated NJ contributions at about $138, consistent with UI/workforce 0.425% plus 2024 FLI 0.09% and no TDI. The 2026 rates add TDI 0.19% and FLI 0.23%, for $226.46 in total."
+us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"Counted only NJ UI and FLI, at $123.80 (about 0.46%). It left out the 0.19% employee TDI and understated FLI and workforce relative to the 2026 total of $226.46 (TDI $50.92, FLI $61.64, UI $102.51, workforce $11.39)."
+us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Applied only 7.65% FICA to the spouse's wages. It never added NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46."
+us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Counted NJ SUI/workforce (0.425%) and FLI at the 2024 rate of 0.09%, for $138.02. It left out the 0.19% employee TDI ($50.92) and understated FLI, whose 2026 rate is 0.23% ($61.64)."
+us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Included only NJ FLI, SUI and workforce, for $139.32. It left out the 0.19% employee TDI and used an FLI rate far below the 2026 rate of 0.23%, so it missed $87.14 of the $226.46 NJ total."
+us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Stopped at Social Security and Medicare ($2,050.20). It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46."
+us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value. The correct figure is $2,050.20 of FICA plus $226.46 of NJ employee contributions, or $2,276.66."
+us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Treated the NJ employee state payroll tax as 'not modeled' and reported FICA only. It left out the $226.46 of NJ employee TDI, FLI, UI and workforce contributions."
+us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its own components sum to $2,050.20 ($1,661.60 + $388.60), but it submitted $4,097, roughly double, which is an arithmetic failure. It also stated that no mandatory state employee payroll tax applies, leaving out NJ's $226.46."
+us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Submitted $1,525, which is below even the 7.65% federal FICA on $26,800 ($2,050.20), so the federal calculation itself is wrong. It also left out the $226.46 of NJ employee TDI, FLI, UI and workforce contributions."
+us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Applied a combined 0.9425% NJ employee rate ($252.59). The 2026 rates sum to 0.845%: UI 0.3825%, workforce 0.0425%, TDI 0.19% and FLI 0.23%. That overstates the NJ contributions by $26.13."
+us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"Its NJ figure of $253.26 is 0.945% of wages, 0.10 percentage points above the 2026 combined rate of 0.845%. That gap equals using the 2025 FLI rate (0.33%) instead of the 2026 rate (0.23%), and overstates FLI by $26.80."
+us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Applied the 2025 NJ rates of TDI 0.23% and FLI 0.33% (0.985% combined, $263.98). The 2026 rates are 0.19% and 0.23%, for a combined 0.845% ($226.46), so NJ is overstated by $37.52."
+us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Estimated NJ employee contributions at $202.34, a combined 0.755% of wages. The 2026 combined rate is 0.845%: UI 0.3825%, workforce 0.0425%, TDI 0.19% and FLI 0.23%, totaling $226.46. That leaves NJ $24.12 short."
+us,scenario_008,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Used the 2025 NJ combined employee rate of 0.985% (TDI 0.23%, FLI 0.33%, UI/workforce 0.425%), giving $263.98. The 2026 rates are TDI 0.19% and FLI 0.23%, which give $226.46."
+us,scenario_008,payroll_tax,gpt-6.1-sol,llm_error,state_local_rule,False,"Its NJ total of $215.07 is exactly $11.39 below the correct $226.46. That is the amount of the 0.0425% workforce development/supplemental workforce fund contribution, so that component was omitted from the four NJ employee contributions."
+us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Applied only 7.65% FICA (rounded to $2,050). It never added NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46."
+us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"Counted only NJ TDI plus FLI, at a combined 0.18% ($48.24). The 2026 TDI and FLI rates are 0.19% and 0.23%, which come to $112.56, and it left out the employee UI (0.3825%, $102.51) and workforce (0.0425%, $11.39) contributions."
+us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Included UI at 0.3825% but estimated TDI and FLI at only about $75 and left out the 0.0425% workforce contribution. Its NJ total of $178 falls short of the $226.46 produced by TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%."
+us,scenario_008,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"Used the 2025 NJ rates of TDI 0.23% and FLI 0.33% (0.985% combined, $263.98). The 2026 rates are TDI 0.19% and FLI 0.23%, a combined 0.845% ($226.46)."
+us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Explicitly assumed no state payroll taxes apply. NJ imposes mandatory employee TDI, FLI, UI and workforce contributions of $226.46 on the spouse's $26,800."
+us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"Counted only NJ UI at 0.3825% and FLI at the 2024 rate of 0.09%. It left out the 0.0425% workforce contribution and the 0.19% employee TDI, and understated FLI, whose 2026 rate is 0.23%."
+us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value. The correct figure is $2,050.20 of FICA plus $226.46 of NJ employee contributions, or $2,276.66."
+us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Estimated NJ UI/workforce, TDI and FLI at $257.95 (about 0.9625%), above the 2026 combined rate of 0.845% ($226.46). It overstated the TDI and FLI rates, which are 0.19% and 0.23% for 2026."
+us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Computed only federal Social Security and Medicare (rounded to $2,050). It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46."
+us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Assumed NJ employee payroll tax is not modeled and reported FICA only. It left out the $226.46 of NJ employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions."
+us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Added only NJ UI at 0.3825% ($102.51) and assumed the 2024 TDI and FLI employee rates of 0%. The 2026 rates are TDI 0.19% ($50.92) and FLI 0.23% ($61.64), and it also left out the 0.0425% workforce contribution ($11.39)."
+us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied only 7.65% FICA to the spouse's wages. It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46."
us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling and housing assistance as sufficient for reduced-price support, without applying the mutually exclusive free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility through SNAP/TANF, the household qualifies for free meals, which makes reduced-price eligibility false."
us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $306, exactly 7.65% of $4,000, despite stating that it applied 15.3%. It effectively used only half the combined Social Security and Medicare self-employment tax rate and also failed to apply the 92.35% statutory earnings factor, instead of calculating $4,000 × 0.9235 × 0.153 = $565.18."
us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated gross self-employment tax of $565.18, then incorrectly subtracted a supposed deduction from that tax liability. The deductible half of self-employment tax reduces adjusted gross income for income-tax purposes; it is not a credit against self-employment tax, so the liability remains $565.18."
-us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a stale $258 standard deduction and the FY2025 $1,756 maximum allotment instead of $299 and $1,789, and its own formula gave $1,214/month. It then replaced that with an unexplained $687/month, blamed on housing assistance disallowing a shelter deduction that it had already set to $0, so the change had no basis in its own derivation."
-us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of the FY2026 $299. It also skipped the SNAP rounding steps: rounding net income to the whole dollar ($1,764) and rounding the 30% contribution up to $530. That gave $1,260.72/month instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It left out the 20% earned income deduction ($513.33), used a $227 standard deduction instead of $299, and used a $1,164 maximum allotment for eight people instead of $1,789. Its $583/month does not even follow from its own inputs."
-us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted the monthly standard deduction (about $280) once from annual income ($24,755 → $24,475) instead of 12 times. That inflated monthly net income to $2,040 instead of $1,764. It also used the FY2025 $1,756 maximum instead of $1,789, then gave $13,620 against its own $13,728."
-us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted a $258 standard deduction once from annual income instead of $299 each month, which inflated net income to $2,041/month instead of $1,764. It used the FY2025 $1,756 maximum instead of $1,789, then cut its own $13,728 to $11,352 with an unspecified 'income/utility' adjustment."
-us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used an $1,802 maximum allotment instead of $1,789 and net income of about $1,700 instead of $1,764, calling on a shelter deduction that does not apply. Its $1,040/month also contradicts its own formula, since $1,802 − 0.3 × $1,700 = $1,292."
-us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"It had every parameter right ($299 standard deduction, $513.33 earned income deduction, $1,789 maximum) but skipped SNAP's rounding. Net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259/month, not $1,259.82."
-us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $1,536 maximum allotment for eight people instead of the FY2026 $1,789, and a $258 standard deduction instead of the $299 for six or more people. That gave $1,536 − $541 = $995/month instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It invented a shelter deduction with a utility allowance and a disability-related deduction, cutting net income to about $8,800/year when the correct figure is $1,764/month ($21,168/year). No utility costs are listed and rent is below half of adjusted income. It also used a $1,691 maximum instead of $1,789, and its $958/month does not match its own formula."
-us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It had net income ($1,764) and the maximum ($1,789) right, but rounded the 30% contribution ($529.20) down to $529 instead of up to the next whole dollar ($530). That gave $1,260/month instead of $1,259."
-us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated net income at about $1,830/month instead of $1,764, which implies a standard deduction below the $299 for six or more people. It also used a maximum allotment of about $1,780 instead of $1,789, which left it at about $1,225/month."
-us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used an $1,900 maximum allotment for eight people instead of the FY2026 $1,789, and a $302 standard deduction instead of $299. It also skipped the rounding of net income and the 30% contribution, which gave $1,371.12/month instead of $1,259."
-us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 $1,756 maximum allotment instead of the FY2026 $1,789. Its net income of $1,783.92 implies a $279 standard deduction instead of $299, which gave $1,220.83/month instead of $1,259."
-us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used an $1,800 maximum allotment instead of $1,789. Its net income of $1,752.92 implies a $310 standard deduction instead of $299. It also skipped the round-up of the 30% contribution to $530, which gave $1,274.12/month."
-us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It adjusted for a shelter deduction that does not apply, since $579 rent is below half of the $1,764 adjusted income and no utility costs are listed. Its $1,275/month implies a contribution of about $514 instead of $530, so its net income is below the correct $1,764."
-us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no computation, and $16,400 ($1,366.67/month) is a guess about $108/month above the correct $1,789 − $530 = $1,259. It never applied the 30% contribution on the $1,764 net income."
-us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,other,False,"Its $1,307/month implies a 30% contribution of only about $482 against the $1,789 maximum, meaning net income of about $1,607 instead of $1,764. It understated net income after the $513.33 earned income and $299 standard deductions, and so overstated the benefit."
-us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added New Jersey's standard utility allowance to rent to create an excess shelter deduction. No utility expense is listed, so shelter costs are only the $579 rent, which is below half of adjusted income. That spurious deduction lowered net income below $1,764 and pushed the benefit to $1,399.75/month instead of $1,259."
-us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation, and its $1,208.33/month matches the FY2025 $1,756 maximum minus a contribution of about $548 (net income near $1,826). The correct inputs are the FY2026 $1,789 maximum and a $530 contribution on $1,764 net income."
-us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $287 standard deduction instead of the FY2026 $299 for six or more people, and the FY2025 $1,756 maximum instead of $1,789. That gave $1,756 − $533 = $1,223/month instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,gemini-3.7-flash,llm_error,other,False,"It gave no computation. Its $1,404/month implies a contribution of only about $385 against the $1,789 maximum, meaning net income near $1,283, roughly $480 less than the correct $1,764. That reflects deductions that do not apply, since there is no shelter or medical deduction here."
-us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It cited a shelter deduction that does not apply, since $579 rent is below half of the $1,764 adjusted income and there are no utility costs. Its $1,316.67/month implies a contribution of about $472 instead of $530, so it understated net income."
-us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 standard deduction once from annual income, instead of the $299 monthly deduction applied each month ($3,588/year). That overstated net income at $24,557/year instead of $21,168. It also used a $1,748 maximum instead of $1,789."
-us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of the $299 for six or more people, which raised net income to $1,853 instead of $1,764. It also used a $1,797 maximum instead of $1,789, which gave $1,241/month instead of $1,259."
-us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned $0 on the claim that housing assistance and earnings push the household past a benefit threshold. Gross income is only 57% of the poverty guideline, NJ categorical eligibility applies, and housing assistance does not bar SNAP. The standard formula gives $1,789 − $530 = $1,259/month."
-us,scenario_008,snap,gpt-5.4-nano,llm_error,other,False,"It returned $0 on the claim that no computation inputs were given, but household size, wages, self-employment income, interest and rent were all provided. Those inputs yield $1,764 net income and a $1,259 monthly benefit."
-us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It applied the right deductions and maximum but gave $1,260/month. The 30% contribution on $1,764 net income ($529.20) rounds up to $530, which yields $1,259/month, not $1,260."
-us,scenario_008,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,139/month implies a contribution of about $650 against the $1,789 maximum, meaning net income near $2,167 instead of $1,764. It under-applied the $513.33 earned income and $299 standard deductions it named, despite also claiming a shelter deduction."
-us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"Its $1,262/month implies a contribution of $527 instead of $530. It did not round net income to $1,764 and round the 30% contribution up to the next whole dollar against the $1,789 maximum, which gives $1,259."
-us,scenario_008,snap,gpt-5.6-terra,llm_error,other,False,"Its $1,284.02/month implies a contribution of about $505 instead of $530, meaning net income near $1,683 instead of $1,764. It overstated the deductions and never rounded the contribution up to a whole dollar."
-us,scenario_008,snap,gpt-6-astra,llm_error,period_annualization,False,"It computed on annual totals ($21,468 maximum less 30% of $21,167 net) and skipped SNAP's monthly rounding. Monthly net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259 × 12 = $15,108, not $15,117.90."
-us,scenario_008,snap,gpt-6-luna,llm_error,other,False,"Its $1,288.17/month implies a contribution of about $501 against the $1,789 maximum, meaning net income near $1,670 instead of $1,764. It overstated the earned income and standard deductions, or the maximum, instead of reaching the $530 contribution."
-us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It truncated net income to $1,763 instead of rounding $1,763.92 to $1,764, and treated the contribution as roughly $529 instead of rounding 30% up to $530. That gave $1,260/month instead of $1,259."
-us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 on the claim that housing assistance eliminates SNAP. Receiving housing assistance does not disqualify the household: it is categorically eligible in NJ with gross income at 57% of the poverty guideline, and its $1,764 net income yields $1,259/month."
-us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a projected maximum of about $21,870/year ($1,822.50/month) instead of $1,789. It also applied a medical deduction from over-the-counter costs, but the spouse's $25/month is below the $35 threshold. Together these gave $1,300/month instead of $1,259."
-us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a $1,759 maximum allotment instead of the FY2026 $1,789. Its net income of $1,772 implies a standard deduction of about $291 instead of $299, which gave $1,227/month instead of $1,259."
-us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used FY2025 parameters: a $1,756 maximum ($21,072/year) and a $291 standard deduction ($3,492/year). The FY2026 values are $1,789 and $299. It also computed annually without the monthly rounding to a $530 contribution."
-us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal $4,250 resource limit for disabled-member households to the $69,500 bank balance and zeroed SNAP. The household is categorically eligible through New Jersey's TANF-funded broad-based categorical eligibility, which removes the asset test."
-us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used an $1,806 maximum allotment instead of the FY2026 $1,789. It also took a $529 contribution instead of rounding 30% of $1,764 up to $530, which gave about $1,276/month instead of $1,259."
-us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value and no explanation, so there is no usable answer to compare with the $15,108 derived from $1,789 − $530 per month."
-us,scenario_008,snap,kimi-k3,llm_error,other,False,"It used the correct $299 standard deduction, $513.33 earned income deduction and $1,789 maximum, but skipped the rounding steps. Net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259/month, not $1,259.825."
-us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied a $3,750 asset limit to the $69,500 bank balance and declared the household ineligible. It ignored that New Jersey's broad-based categorical eligibility (TANF non-cash) waives the SNAP asset test."
-us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used an $1,795 maximum allotment instead of the FY2026 $1,789, and a $297 standard deduction instead of $299. That gave $1,265/month instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of $299 and an $1,838 maximum instead of $1,789, and left the $115 of interest out of net income. It then cut its own $15,391 to $14,784 on an unsupported housing-assistance shelter adjustment, even though no shelter deduction applies."
-us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed $2,567/month exceeds 130% of the poverty guideline for eight people. That limit is about $5,900/month, and the household's gross income is only 57% of the guideline, so it passes the gross income test and receives $1,259/month."
+us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used outdated parameters: a $258 standard deduction instead of the FY2026 $299 for 6+ people, and a $1,756 maximum allotment instead of $1,789. It computed $1,214/month and then replaced that with an unsupported $687/month. Correct inputs give $1,763.92 net income (rounded to $1,764), a $530 contribution and $1,259/month."
+us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of the FY2026 $299 for 6+ people, so net income came out at $1,760.92 instead of $1,763.92. It also did not round net income to $1,764 or round the 30% contribution up to $530, which produced $1,260.72/month instead of $1,259."
+us,scenario_008,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the 20% earned income deduction ($513.33/month) and used a $227 standard deduction instead of $299, which overstated net income at $2,349. It also used a $1,164 maximum allotment, far below the FY2026 $1,789 for eight people."
+us,scenario_008,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It denied SNAP because of the $69,500 in bank assets. The household is categorically eligible through New Jersey's TANF non-cash (BBCE) pathway, which removes the asset test, and its gross income is only 57% of the poverty guideline."
+us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted a monthly standard deduction (about $280) once from annual income instead of $299 every month. That overstated monthly net income at $2,040 instead of $1,764. It also used the outdated $1,756 maximum allotment and then gave an arbitrary $13,620 that differs from its own $13,728."
+us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted the $258 monthly standard deduction once from annual income and used $258 instead of the FY2026 $299, so monthly net income came out at $2,041 instead of $1,764. It also used the outdated $1,756 maximum allotment and then cut its own $13,728 result to an unsupported $11,352."
+us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used a $1,802 maximum allotment instead of $1,789. Its numbers also contradict each other: $1,802 minus 30% of $1,700 is $1,292, not $1,040. The correct net income of $1,764 gives a $530 contribution and $1,259/month."
+us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were right, but it did not round net income to the whole dollar ($1,764) or round the 30% contribution up to the next whole dollar ($530). That gave $1,259.82/month instead of $1,259."
+us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $1,536 maximum allotment instead of the FY2026 $1,789 for eight people. It also used a $258 standard deduction instead of $299 for 6+ people, which produced $995/month instead of $1,259."
+us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up an excess shelter deduction with a utility allowance even though no utility costs were listed, and it applied a disability deduction. That pushed net income down to about $8,800/year. The $579 rent is below half of the $1,764 net income, so there is no shelter deduction. It also used a $204 standard deduction and a $1,691 maximum allotment, and its own arithmetic gives about $17,652, not $11,500."
+us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It correctly found $1,764 net income and the $1,789 maximum allotment, but it did not round the 30% contribution ($529.20) up to $530. As a result it rounded the benefit to $1,260/month instead of $1,259."
+us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated net income at about $1,830 instead of $1,764 even while claiming a shelter deduction that does not apply. It also used about $1,780 as the maximum allotment instead of $1,789."
+us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used about $1,900 as the maximum allotment instead of the FY2026 $1,789 for eight people, and a $302 standard deduction instead of $299. It also skipped the rounding of the 30% contribution, which overstated the benefit at $1,371/month."
+us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 instead of the FY2026 $1,789. Its net income of $1,783.92 implies a standard deduction of about $279 instead of $299, which produced $1,220.83/month instead of $1,259."
+us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used an $1,800 maximum allotment instead of $1,789. Its net income of $1,752.92 implies a standard deduction of about $310 instead of $299. It also did not round net income or round the contribution up."
+us,scenario_008,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"$15,300 works out to $1,275/month, or $16 above the correct $1,259 ($1,789 minus a $530 contribution on $1,764 net income). That matches an inflated maximum allotment or a shelter deduction it invoked. No shelter deduction applies because the $579 rent is below half of net income."
+us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no calculation. $16,400 ($1,367/month) is about $108/month above the correct $1,259, which matches an inflated maximum allotment or understated net income rather than $1,789 minus 30% of $1,764 rounded up to $530."
+us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,307/month is $48 above the correct $1,259. That matches a maximum allotment higher than the FY2026 $1,789, or net income understated by about $160 below the correct $1,764 (which carries a $530 contribution)."
+us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added New Jersey's standard utility allowance to rent to create an excess shelter deduction, even though no utility expenses were listed. Without it, the $579 rent is below half of the $1,764 net income, so the shelter deduction is $0 and the benefit is $1,259/month."
+us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation. $14,500 ($1,208/month) is $51/month below the correct $1,259, which matches the outdated $1,756 FY2025 maximum allotment rather than the FY2026 $1,789 minus a $530 contribution."
+us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 instead of the FY2026 $1,789, and a $287 standard deduction instead of $299. That gave $1,775.92 net income and $1,223/month instead of $1,259."
+us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no calculation. $16,848 ($1,404/month) is $145/month above the correct $1,259, which matches a much higher maximum allotment or deductions not in the facts, not $1,789 minus 30% of $1,764 rounded up to $530."
+us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction that does not exist here: the $579 rent is below half of the $1,764 net income. The resulting $1,317/month is above the correct $1,259 ($1,789 minus $530)."
+us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 standard deduction once from annual income instead of $299 every month, which overstated net income at $24,557/year. It also used a $1,748 maximum allotment instead of $1,789."
+us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of the FY2026 $299 for households of 6+, so net income came out at about $1,853 instead of $1,764. It also used a $1,797 maximum allotment instead of $1,789."
+us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It gave $0 with no supporting test. The household's gross income is 57% of the poverty guideline, it passes both income tests, and it is categorically eligible through New Jersey BBCE. Housing assistance does not disqualify it, so it receives $1,259/month."
+us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It refused to compute and gave $0, even though the facts provided every input needed. With $2,576.25 gross income, $299 standard and $513.33 earned income deductions, and categorical eligibility through BBCE, the benefit is $1,259/month."
+us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It used the right deductions and maximum allotment, but it did not round the 30% contribution ($529.20) up to $530. As a result it rounded the benefit to $1,260/month instead of $1,259."
+us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $1,139/month is $120 below the correct $1,259 and implies net income of about $2,167 against the $1,789 maximum allotment. That means it understated the standard and earned income deductions, which total $812.33 and give $1,764 net income. It also claimed a shelter deduction that does not apply."
+us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"Its $1,262/month implies a contribution of about $527 instead of $530. It did not round net income to $1,764 and round 30% of it up to $530, so the result is $3/month above $1,259."
+us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,284.02/month implies a contribution of about $505, which corresponds to net income of about $1,683 instead of $1,764. That means it overstated deductions beyond the $299 standard and $513.33 earned income deductions, and it also skipped rounding the contribution up."
+us,scenario_008,snap,gpt-6-astra,llm_error,other,False,"It computed on annual totals ($21,468 minus 30% of $21,167) without the monthly rounding steps. Net income needs to be rounded to $1,764 and the 30% contribution rounded up to $530 each month, which gives $1,259/month, not $1,259.825."
+us,scenario_008,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $1,288/month is $29 above the correct $1,259. That matches a maximum allotment higher than the FY2026 $1,789, or net income understated below the correct $1,764 (which carries a $530 contribution)."
+us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It cut net income of $1,763.92 down to $1,763 instead of rounding to $1,764. It also used a 'roughly $529' contribution instead of rounding 30% up to $530, which produced $1,260/month instead of $1,259."
+us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It wrongly treated housing assistance and income as making the household ineligible. Subsidized housing does not bar SNAP, gross income is 57% of the poverty guideline, and BBCE makes the household eligible for $1,259/month."
+us,scenario_008,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It gave an excess medical deduction for the disabled spouse's over-the-counter costs, but $300/year is $25/month, below the $35 threshold, so no medical deduction applies. It also used about $1,822.50/month as the maximum allotment instead of $1,789."
+us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used outdated parameters: about $1,759 as the maximum allotment instead of the FY2026 $1,789, and a standard deduction near the FY2025 $291 instead of $299. That gave $1,227/month instead of $1,259."
+us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly carried over FY2025 parameters: a $1,756/month maximum allotment ($21,072/year) and a $291/month standard deduction ($3,492/year). FY2026 uses $1,789 and $299, with net income rounded to $1,764 and the contribution rounded up to $530."
+us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal $4,250 resource limit for households with a disabled member and denied benefits over the $69,500 in bank assets. New Jersey BBCE (TANF non-cash categorical eligibility) waives the asset test entirely."
+us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used an $1,806 maximum allotment instead of the FY2026 $1,789 for eight people. It also used a $529 contribution instead of rounding 30% of $1,764 up to $530."
+us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value and no explanation, so there was no answer to score."
+us,scenario_008,snap,kimi-k3,llm_error,other,False,"Its inputs were right ($1,763.92 net income, $1,789 maximum allotment), but it did not round net income to $1,764 or round the 30% contribution up to $530. That gave $1,259.825/month instead of $1,259."
+us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied an asset limit (and an outdated $3,750 one) to deny SNAP over $69,500 in bank assets. New Jersey BBCE removes the asset test for this household."
+us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,795 maximum allotment instead of $1,789 and a $297 standard deduction instead of $299. It also skipped rounding the 30% contribution up to $530, which gave $1,265/month instead of $1,259."
+us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of the FY2026 $299 for 6+ people and a $1,838 maximum allotment instead of $1,789. It also left out the $115 interest and then cut the result by an unsupported 'housing assistance' adjustment."
+us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It wrongly found that $2,567/month exceeds 130% of the poverty guideline for eight people. That limit is roughly $5,900/month, and the household's gross income is only 57% of the poverty guideline, so it passes the gross income test easily."
us,scenario_008,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Declared the spouse ineligible ""under adult NJ rules"" without running New Jersey's ACA expansion income test, which covers adults 19-64 up to 138% FPL and requires no categorical hook such as pregnancy or caretaker status. Household MAGI of about $30,900 against the eight-person FPL is 0.55 x FPL, far inside that limit; the bare denial is consistent with comparing the spouse's own $26,800 of wages to a one- or two-person 138% FPL line (roughly $22,600 to $30,700) instead of household MAGI over a family size of eight."
us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to 0 by treating Medicaid eligibility as something requiring an affirmative listed trigger fact, extending the prompt's ""treat unlisted inputs as false"" rule to the computed output itself. Every input the determination needs was supplied - age 41, non-dependent, New Jersey, household income - and the MAGI adult-expansion test it never ran places household MAGI at 0.55 x FPL against the 138% FPL threshold, which returns eligible."
us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Ran the MAGI test correctly - about $31,420 of household income against the roughly $73,637 138%-FPL line for eight people - then reversed its own correct result with a fabricated rule that receipt of housing assistance disqualifies adults from Medicaid. Housing assistance is excluded from MAGI and disqualifies no one from any Medicaid category, and the MAGI adult category applies no resource test, so neither the housing subsidy nor the $69,500 in bank accounts affects eligibility."
@@ -696,218 +699,227 @@ us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligib
us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred breastfeeding or postpartum status and nutritional risk solely from the presence of a one-year-old, despite the prompt requiring unlisted statuses to be false. With neither pregnancy, breastfeeding, nor postpartum status listed, the spouse fails WIC's categorical test regardless of income."
us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical qualifier. Disability and income eligibility do not establish WIC eligibility for an adult who is not pregnant, breastfeeding, or postpartum."
us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse two separate $1,000 exemptions for blind and disabled, so exemptions came to $13,000 instead of $12,000. It then subtracted the 18%-of-rent property tax deduction ($1,251). The household takes the larger $50 refundable property tax credit instead, so no rent deduction reduces taxable income. Taxable income should be $18,915, taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it also subtracted the renter property tax deduction of 18% × $6,949 = $1,250.82. That deduction saves only $17.51 at 1.4%, less than the $50 refundable property tax credit, so the credit is taken instead and the deduction is not. Taxable income stays at $18,915 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Applied the federal American Opportunity Credit, a nonexistent NJ standard deduction and a pension exclusion to NJ tax, and concluded taxable income was zero. NJ gives no AOC and no standard deduction. The only reductions are $12,000 of exemptions, leaving $18,915 taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Computed about $251 of tax (using $13,000 of exemptions, which double-counted blind and disabled). It then rounded that to $0 on the basis of unnamed nonrefundable credits, although no NJ nonrefundable credit applies. The correct $12,000 of exemptions leaves $18,915 taxable and $264.81 of tax."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Said $30,900 of gross income was below the $20,000 MFJ filing threshold, which is backwards: $30,915 is above it, so the household owes tax. After $12,000 of exemptions, $18,915 is taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used $1,500 per spouse for the regular exemption ($3,000) when NJ allows $1,000 each ($2,000). That pushed total exemptions to $13,000 and taxable income down to $17,915, giving $251 instead of 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse separate $1,000 exemptions for blind and disabled ($13,000 total instead of $12,000). It then invented a low-income phase-in that cuts liability to zero. NJ's only low-income relief is the $20,000 MFJ gross-income threshold, which this household exceeds, so tax is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Claimed an NJ MFJ standard deduction larger than $30,915. NJ has no standard deduction. Only $12,000 of exemptions apply, leaving $18,915 taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $30,632, which subtracts half of SE tax, a deduction NJ does not allow; NJ gross income is $30,915. It also allowed only $9,000 of exemptions, leaving out the $2,000 of spouse exemptions and the $1,000 blind/disabled exemption. That wrongly pushed taxable income into the 1.75% bracket."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it took a $181.70 medical deduction built from over-the-counter health expenses. Those expenses don't qualify for NJ's medical deduction, so taxable income is $18,915 and tax is $264.81, not $262.27."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,age_disability,False,"Allowed only $11,000 of exemptions, leaving out the $1,000 exemption for the blind and disabled spouse. That gave $19,915 taxable instead of $18,915 and tax of $278.81 instead of $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Correctly reached $18,915 of taxable income and about $265 of tax, then subtracted a $50 tenant property tax credit it treated as nonrefundable. NJ's property tax credit is refundable, so it does not reduce tax before refundable credits, which is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Claimed personal exemptions eliminate the tax. Total NJ exemptions are only $12,000 against $30,915 of gross income, leaving $18,915 taxable at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its preliminary tax of about $251 implies $13,000 of exemptions (blind and disabled counted separately) instead of $12,000. It then subtracted the $50 property tax credit, which is refundable and does not reduce tax before refundable credits. Correct tax is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Overstated exemptions at $14,000. It used $3,000 for the MFJ personal exemptions instead of $2,000, and $2,000 for the spouse being blind and disabled instead of a single $1,000. That left $16,915 taxable instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Reported $0 with no derivation. NJ gross income of $30,915 less $12,000 of exemptions leaves $18,915 taxable at 1.4%, so tax is $264.81; the $0 answer implies it wrongly assumed exemptions or credits wipe out the tax."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Correctly computed $264.81 on $18,915 of taxable income, then subtracted a $50 property tax credit. NJ's property tax credit is refundable, so it belongs below this line, and tax before refundable credits stays at $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,age_disability,False,"Gave the spouse separate $1,000 blind and disabled exemptions ($13,000 total). NJ allows a single $1,000 exemption for a person who is blind or disabled, so exemptions are $12,000 and tax is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Valued all eight people's exemptions at $1,000 ($8,000), when dependents get $1,500 each. It also invented a $4,000 MFJ standard deduction, which NJ doesn't have, and subtracted the 18%-of-rent property tax deduction, which isn't taken because the $50 refundable credit is worth more. The correct figure is $12,000 of exemptions, $18,915 taxable and $264.81 of tax."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed exemptions and credits wipe out the tax without computing them. $12,000 of exemptions leaves $18,915 taxable, no nonrefundable NJ credit applies, and tax is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Assumed deductions and credits reduce the tax to zero. NJ allows only $12,000 of exemptions here and no nonrefundable credits, so $18,915 is taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction built from over-the-counter health expenses, which don't qualify for NJ's medical deduction. Taxable income is therefore $18,915, not $18,733.30, and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Used the NJ Child Tax Credit to offset tax before refundable credits. The NJ CTC is refundable, so it doesn't reduce this amount, which stays at 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Took a medical deduction built from over-the-counter health expenses, which don't qualify. It also subtracted the renter property tax credit as if it were nonrefundable. The NJ property tax credit is refundable, and no medical deduction applies, so tax before refundable credits is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"Counted blindness and disability as two separate $1,000 exemptions for the spouse, which implies $13,000 of exemptions and $250.81 of tax. NJ allows a single $1,000 blind-or-disabled exemption per person, so taxable income is $18,915 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it subtracted an estimated $181.70 medical deduction built from over-the-counter health expenses, which don't qualify for NJ's medical deduction. Taxable income is $18,915 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Correctly computed $264.81 on $18,915, then subtracted a $50 renter property tax credit. That credit is refundable in NJ, so it doesn't reduce tax before refundable credits, which is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Relied on an NJ standard deduction that doesn't exist, plus unspecified credits. Only $12,000 of exemptions apply, leaving $18,915 taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used federal AGI of $30,632 (after the half-SE-tax deduction) instead of NJ gross income of $30,915. It also valued the six dependent exemptions at $1,000 each instead of $1,500, for $9,000 of exemptions instead of $12,000. That pushed taxable income into the 1.75% bracket."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Correctly reached $18,915 taxable and about $265 of tax, then subtracted the $50 renter property tax credit as if it were nonrefundable. NJ's property tax credit is refundable, so tax before refundable credits is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Treated $800 of over-the-counter health expenses as deductible medical costs above 2% of gross income, a $181.70 deduction. OTC expenses don't qualify for NJ's medical deduction, so taxable income is $18,915 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Left out the spouse's $1,000 blind/disabled exemption, giving $11,000 of exemptions. It then taxed the whole $19,915 at 1.75%, although the MFJ schedule taxes the first $20,000 at 1.4%. Correct tax is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Understated exemptions at $10,000 instead of $12,000. It also subtracted a $1,251 renter property tax deduction, which isn't taken because the $50 refundable property tax credit is worth more. Taxable income is $18,915 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Correctly reached $12,000 of exemptions, $18,915 taxable and $264.81 of tax, then subtracted the $50 renter property tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits, which is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $30,632, which subtracts half of SE tax, instead of NJ gross income of $30,915. It also left out the spouse's $1,000 blind/disabled exemption ($11,000 instead of $12,000), leaving $19,632 taxable instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction built from over-the-counter health expenses, which don't qualify, and subtracted the $50 renter property tax credit, which is refundable. Without either, tax before refundable credits is 1.4% × $18,915 = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Valued the dependent exemptions at $1,000 instead of $1,500. It then zeroed the tax with an invented nonrefundable exemption reduction for low-income households, a provision NJ doesn't have. With $12,000 of exemptions, $18,915 is taxed at 1.4% = $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Said taxable income is zero after exemptions and deductions. NJ exemptions total only $12,000 against $30,915 of gross income, leaving $18,915 taxable at 1.4% = $264.81."
-us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the federal EITC down to about $6,110, but earned income of about $30,500 is below the 2026 MFJ phase-out start, so the full $8,231 applies and the NJ EITC is $3,292.40. It then gave a muddled NJ CTC of $800 based on gross income. The June 2026 schedule applied to taxable income under $30,000 gives $2,500 for the two children under 6. It also left out the $50 tenant property tax credit."
-us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the old $1,000-per-child NJ CTC. The schedule enacted June 30, 2026 for 2026–2028 pays $2,500 for the two children under 6 at this taxable income. It also left out the $50 refundable property tax credit available to renters with $6,949 of listed rent."
-us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It said New Jersey has no refundable income tax credits. In fact the NJ EITC (40% of federal, $3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit are all refundable and all apply to this household."
-us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own reasoning added up to about $3,880 ($2,880 EITC plus $1,000 CTC), yet it submitted $9,492, which none of its components support. Its steps were also wrong in three ways. It phased the federal EITC below the $8,231 maximum. It counted only the age-1 child for the NJ CTC, although the age-5 child also qualifies. And it left out the June 2026 schedule's $2,500 CTC and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The $3,292 EITC is correct, but it cut the NJ CTC to $600 per child 'near $30,500' using gross income. NJ taxable income after exemptions is about $18,900, and the 2026 schedule pays $2,500 for the two children under 6. It then padded the total with unexplained 'minor amounts' rather than the specific $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It said the NJ Child Tax Credit does not apply at this income. With NJ taxable income of about $18,900, the credit pays $2,500 for the children aged 5 and 1. It also understated the federal EITC as $7,700 instead of the full $8,231, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct, but it used the pre-2026 NJ CTC of $1,000 per child under 6. The schedule enacted June 30, 2026 pays $2,500 for the two young children at this taxable income, so its answer is $500 short."
-us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used an outdated $7,830 EITC maximum and began the phase-out at about $25,511, which is not the MFJ threshold. That cut the federal EITC to $6,716, when the correct 2026 figure is the full $8,231 (NJ EITC $3,292.40). It also used a $500-per-child NJ CTC instead of the 2026 schedule's $2,500 total, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It guessed the federal EITC at $8,000–$9,000 and the NJ EITC at about $3,800, overstating the exact 40% × $8,231 = $3,292.40. It used the old $1,000-per-child CTC instead of the 2026 schedule's $2,500. It also left out the $50 property tax credit, so its total is a rough sum rather than a computed one."
-us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It put the family in the $30,001–$40,000 CTC bracket ($800 per child) using gross income. The NJ CTC uses NJ taxable income, which is about $18,900 after $12,000 of exemptions, and the June 2026 schedule pays $2,500 at that level. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only the NJ EITC, on an understated federal EITC of $8,021 instead of the full $8,231. It left out the refundable NJ Child Tax Credit ($2,500 for the children aged 5 and 1) and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a federal EITC of $7,787, when the full $8,231 applies. It left out the $2,500 refundable NJ Child Tax Credit for the two children under 6 and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a federal EITC of $8,190 instead of the 2026 maximum of $8,231, and the pre-2026 NJ CTC of $1,000 per child instead of the June 2026 schedule's $2,500. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used the old $1,000-per-child NJ CTC instead of the 2026 schedule's $2,500 for two children under 6. It also made up a $75 renters' credit; the actual NJ property tax credit for tenants is $50. And its federal EITC base of $8,247 overstates the $8,231 maximum."
-us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It understated the federal EITC as $7,691 instead of the full $8,231. It then put the NJ CTC in the $800 tier using gross income, but the tiers use NJ taxable income (about $18,900), and the June 2026 schedule pays $2,500 there. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no refundable NJ credits. This low-income MFJ family with six children receives the refundable NJ EITC ($3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It put the family in the $800-per-child NJ CTC tier using gross income, instead of the lowest taxable-income tier of the June 2026 schedule, which gives $2,500. It also left out the $50 property tax credit, and its EITC of about $3,280 is slightly below $3,292.40."
-us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It phased the federal EITC down to $7,128, but this household is below the 2026 MFJ phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It also used $800 per child in CTC instead of the 2026 schedule's $2,500 total, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave $1,200 with no calculation. The correct figure is $5,842.40: a $3,292.40 NJ EITC (40% of $8,231), a $2,500 NJ CTC for the two children under 6, and a $50 property tax credit. Its $1,200 does not come from any of these."
-us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It used a $1,600 NJ CTC ($800 per child, the gross-income tier) instead of the June 2026 schedule's $2,500 at NJ taxable income of about $18,900. Its EITC of $3,219.60 implies a federal EITC of $8,049, below the $8,231 maximum. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,776 fits a slightly understated NJ EITC plus a $1,600 CTC at $800 per child. That misses the June 2026 schedule's $2,500 for the two children under 6, which uses taxable income, and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right, but it used an $800-per-child NJ CTC based on gross income. The correct amount is $2,500, from the June 2026 schedule applied to NJ taxable income of about $18,900."
-us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there is nothing to parse."
-us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It counted only one child under 6 for the NJ CTC, but children aged 5 and younger qualify, so both the 5-year-old and the 1-year-old count. It also used $500 instead of the 2026 schedule's $2,500 total. It overstated the federal EITC as $8,360 instead of $8,231, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no refundable NJ credit applies. This household qualifies for the refundable NJ EITC ($3,292.40 from its $8,231 federal EITC), the NJ Child Tax Credit ($2,500), and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It returned $0, saying low-income status alone creates no refundable NJ credits. The NJ EITC and the NJ CTC are exactly low-income refundable credits, and together with the $50 property tax credit they total $5,842.40 here."
-us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the pre-2026 NJ CTC of $1,000 per child. The schedule enacted June 30, 2026 gives $2,500 for the two young children. It also left out the $50 refundable property tax credit for renters."
-us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $8,046 federal EITC. It left out the refundable $2,500 NJ Child Tax Credit for the children aged 5 and 1 and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used $1,600 of NJ CTC ($800 per child, the gross-income tier) instead of the June 2026 schedule's $2,500 at NJ taxable income under $30,000. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 equals the correct $3,292.40 EITC plus a $1,600 CTC. That CTC uses the $800-per-child gross-income tier instead of the 2026 schedule's $2,500, and the total leaves out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are correct, but it used a $1,600 NJ CTC at $800 per child, which puts the family in a tier using gross income. The correct amount is $2,500, from the June 2026 schedule applied to NJ taxable income of about $18,900."
-us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It used an NJ EITC rate of 50% instead of the statutory 40%, which overstates that credit ($4,081 vs $3,292.40). It also used the old $1,000-per-child CTC instead of the 2026 schedule's $2,500, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It overstated the NJ EITC at $3,337 instead of $3,292.40. It used $800 per child in CTC instead of the June 2026 schedule's $2,500 at NJ taxable income, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct, but it used the pre-2026 NJ CTC of $1,000 per child. The schedule enacted June 30, 2026 for 2026–2028 pays $2,500 for the two children under 6, so its answer is $500 short."
-us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It said no state refundable credits apply. The household receives the refundable NJ EITC ($3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It picked the $400-per-child CTC tier using gross income ($30,001–$40,000), but the tiers use NJ taxable income, which is about $18,900 after exemptions, and the June 2026 schedule pays $2,500 at that level. It also left out the $50 property tax credit and slightly overstated the federal EITC as $8,247."
-us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $500-per-child NJ CTC instead of the June 2026 schedule's $2,500 for the two children under 6 at this taxable income. It also said no other refundable credits apply, which leaves out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It used the old $1,000-per-child NJ CTC instead of the 2026 schedule's $2,500. It also refused the $50 property tax credit on the grounds that it would not infer rent, but the spouse's $6,949 pre-subsidy rent is listed and makes the household eligible as a tenant."
-us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a $7,901 federal EITC instead of the full $8,231, and said there are no other refundable credits. That leaves out the $2,500 NJ Child Tax Credit and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an overstated $8,271 federal EITC. It left out the refundable NJ Child Tax Credit ($2,500 for the children aged 5 and 1) and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there is nothing to parse."
-us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the old $1,000-per-child NJ CTC instead of the June 2026 schedule's $2,500 for two children under 6. It also left out the $50 property tax credit for renters."
-us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no NJ refundable credits apply. The household's earned income and young children qualify it for the NJ EITC ($3,292.40) and the NJ CTC ($2,500), and its rent qualifies it for the $50 property tax credit."
-us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It treated the NJ CTC as phasing out continuously on gross income above $30,000. The credit is a step schedule based on NJ taxable income, and at about $18,900 the June 2026 schedule pays $2,500. It also overstated the federal EITC as $8,262 and left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It said the federal EITC is near $0 at about $30,800 of earned income. That is wrong: the household is below the 2026 MFJ phase-out start and gets the full $8,231, so the NJ EITC is $3,292.40. It also counted only one child under 6 and zeroed out the NJ CTC ($2,500) and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It made up an NJ EITC income phase-out and a $100 minimum credit. The NJ EITC is simply 40% of the federal EITC ($3,292.40 here), with no separate state phase-out. It also left out the $2,500 NJ CTC and the $50 property tax credit."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse two $1,000 exemptions, one for blind and one for disabled, for $13,000 total instead of $12,000. NJ allows one blind-or-disabled exemption per person. It then also subtracted an 18%-of-rent property-tax deduction ($1,251) that doesn't enter taxable income, so it taxed $16,664 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Got the $12,000 in exemptions right but then subtracted a $1,250.82 renter property-tax deduction (18% of rent). The household takes the $50 refundable property-tax credit instead of the deduction, so taxable income stays at $18,915 and tax is $264.81, not $247.30."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Zeroed NJ tax by applying the federal American Opportunity Credit, a pension exclusion and a standard deduction, none of which reduce NJ Gross Income Tax here. NJ taxable income is $30,915 minus $12,000 in exemptions = $18,915, taxed at 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Said exemptions eliminate NJ tax without doing the math. The $12,000 in exemptions ($2,000 personal, $9,000 dependents, $1,000 blind/disabled) leave $18,915 of taxable income, which produces $264.81 at the 1.4% rate."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Computed about $251 after double-counting the spouse's blind and disabled exemptions. It then arbitrarily rounded the liability to $0 by citing nonrefundable credits that don't apply. The correct result is $18,915 × 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Misapplied the NJ $20,000 MFJ filing threshold: it said $30,915 of gross income was below it, when that income is above the threshold. With no exemption from tax, the $12,000 in exemptions leave $18,915 taxable at 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used $1,500 personal exemptions for each spouse when NJ's regular exemption is $1,000 per filer ($2,000 for MFJ). That inflated exemptions to $13,000 and cut taxable income to $17,915 (tax $250.81) instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Gave the spouse separate $1,000 exemptions for blind and for disabled (NJ allows only one) to get about $251. It then invented a low-income phase-in that zeroes tax for MFJ filers above the $20,000 threshold, which NJ doesn't have. The correct tax is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Applied an 'NJ MFJ standard deduction' that wiped out income, but NJ has no standard deduction. Only exemptions ($12,000) reduce the $30,915, leaving $18,915 taxed at 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from a federal-style AGI of $30,632 by deducting half of SE tax, which NJ Gross Income doesn't allow, so it should start from $30,915. It also allowed only $9,000 of exemptions, valuing dependents at $1,000 instead of $1,500 each. That pushed taxable income into the 1.75% bracket ($21,632) instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Took a $181.70 NJ medical expense deduction using the $800 of over-the-counter health expenses above 2% of income. Nonprescription OTC items aren't qualifying medical expenses, so taxable income is $18,915 (tax $264.81), not $18,733.30."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Left out the spouse's $1,000 blind-or-disabled exemption and used $11,000 instead of $12,000, so it taxed $19,915 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Got $18,915 of taxable income and about $265 of tax right, then subtracted the $50 NJ property-tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits, which is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Claimed exemptions zero out NJ tax without computing them. $30,915 minus $12,000 in exemptions leaves $18,915 of taxable income and $264.81 of tax at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its preliminary $251 tax reflects $13,000 of exemptions: it gave the spouse separate blind and disabled exemptions, which NJ caps at one. It then subtracted the $50 property-tax credit, which is refundable and excluded from this output."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Used $14,000 of exemptions: $1,500 per spouse instead of the $1,000 regular exemption, plus two $1,000 exemptions for the spouse being both blind and disabled instead of one. That taxed $16,915 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Answered $0 with no derivation. NJ income of $30,915 minus $12,000 in exemptions leaves $18,915 taxable, which produces $264.81 at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Computed $264.81 on $18,915 correctly, then subtracted the $50 NJ property-tax credit. That credit is refundable, so it isn't part of tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave separate $1,000 blind and disabled exemptions for the spouse, for $13,000 total. NJ allows a single blind-or-disabled exemption per person, so total exemptions are $12,000 and taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable value for state_income_tax_before_refundable_credits, so there is no substantive computation to grade."
+us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Subtracted a made-up $4,000 MFJ standard deduction (NJ has none) and an 18%-of-rent property-tax deduction that isn't taken. It also undercounted exemptions at $9,000 by valuing dependents at $1,000 instead of $1,500, reaching $16,664 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed NJ exemptions and credits erase the tax without computing them. $12,000 in exemptions leave $18,915 of taxable income, and no nonrefundable credit offsets the resulting $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Estimated $0 without computing the base. NJ gross income of $30,915 minus $12,000 in exemptions leaves $18,915, taxed at 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Took a $181.70 NJ medical deduction based on over-the-counter health expenses, which aren't qualifying medical expenses. Without it, taxable income is $18,915 and tax is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Used the NJ Child Tax Credit to offset the tax, but that credit is refundable and doesn't belong in tax before refundable credits. Tax on $18,915 at 1.4% is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted an OTC-based medical deduction that isn't allowed, then also netted the $50 renter property-tax credit, which is refundable. Both cuts are wrong. Tax before refundable credits is $18,915 × 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $250.81 reflects $13,000 of exemptions, counting separate blindness and disability exemptions for the spouse. NJ allows only one $1,000 blind-or-disabled exemption per person, so taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Correctly excluded the refundable property-tax credit, but still took a $181.70 medical deduction from over-the-counter health expenses, which don't qualify. Taxable income is $18,915, not $18,733.30."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Computed $264.81 correctly, then subtracted the $50 renter property-tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Used a 'NJ standard deduction' that doesn't exist. Only the $12,000 in exemptions apply, leaving $18,915 taxable and $264.81 of tax."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Equated NJ gross income with federal AGI ($30,632), deducting half of SE tax, which NJ doesn't allow. It also valued dependents at $1,000 instead of $1,500 for $9,000 of exemptions, which pushed taxable income to $21,632 and into the 1.75% bracket."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Correctly picked the $50 property-tax credit over the deduction but subtracted it from tax. That credit is refundable and excluded from this output, which is $18,915 × 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Deducted $181.70 of NJ medical expenses from the $800 of over-the-counter health costs. Nonprescription OTC items aren't qualifying medical expenses, so taxable income stays at $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Left out the spouse's $1,000 blind-or-disabled exemption ($11,000 instead of $12,000). It also applied the 1.75% rate to all of its $19,915 even though NJ MFJ taxes the first $20,000 at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Understated exemptions at $10,000 instead of $12,000 ($2,000 personal + $9,000 dependents at $1,500 each + $1,000 blind/disabled). It then subtracted a $1,251 renter property-tax deduction that doesn't enter taxable income."
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value for state_income_tax_before_refundable_credits, so there is no substantive computation to grade."
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Computed $264.81 correctly, then subtracted the $50 renter property-tax credit. That credit is refundable and doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Started from a federal-style AGI of $30,632, deducting half of SE tax, which NJ Gross Income doesn't allow. It also left out the spouse's $1,000 blind-or-disabled exemption, giving $19,632 of taxable income instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction from over-the-counter health expenses, which don't qualify, and then subtracted the $50 renter property-tax credit, which is refundable. Neither belongs in the computation, which gives $18,915 × 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Valued dependents at $1,000 instead of $1,500 and got about $296. It then invented a 'nonrefundable exemption reduction' that zeroes the tax, which NJ doesn't have. With $12,000 of exemptions, tax is $18,915 × 1.4% = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted that NJ taxable income is zero after exemptions, but $12,000 of exemptions against $30,915 of NJ gross income leaves $18,915 taxable, which produces $264.81."
+us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the federal EITC down to about $6,110, but earned income of about $30.5k is below the 2026 MFJ phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It then gave a $400-per-child NJ CTC based on gross-income brackets, although NJ taxable income is well under $30,000. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The NJ EITC is right at $3,292.40, but the model left out the $50 NJ property tax credit even though the household pays $6,949 in rent. It also used the superseded $1,000-per-child CTC ($2,000) instead of the June 30, 2026 schedule, which pays $2,500 for the two children age 5 and under."
+us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed New Jersey has no refundable income tax credits. NJ's EITC (40% of federal, $3,292.40), child tax credit ($2,500) and $50 property tax credit are all refundable and all apply to this low-income family of eight."
+us,scenario_008,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It said no NJ refundable credits apply. The household qualifies for the refundable NJ EITC ($3,292.40 = 40% of the $8,231 federal EITC), the NJ child tax credit for two children under 6 ($2,500), and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own working reached about $3,880: a phased-down EITC of about $7,200 instead of the full $8,231, plus a CTC for only the age-1 child, which misses the age-5 child. It then submitted an unrelated $9,492. The correct sum is $3,292.40 EITC + $2,500 CTC + $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The NJ EITC of $3,292 is right. The model then cut the CTC to about $1,200 by applying a $600 tier at roughly $30,500 of gross income, but the NJ CTC tier depends on NJ taxable income, which is far below $30,000 after exemptions. It filled the rest of its total with unexplained 'minor amounts' instead of the $50 property tax credit and the $2,500 CTC under the June 2026 schedule."
+us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It used a federal EITC of about $7,700 instead of the $8,231 maximum, which is unreduced at this earned income. It also wrongly said the NJ child tax credit does not apply, when the two children age 5 and under qualify for $2,500 at this low NJ taxable income. It left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It correctly computed the $3,292.40 NJ EITC and the $50 property tax credit, but used the pre-2026 NJ CTC of $1,000 per child under 6 ($2,000). The schedule New Jersey enacted on June 30, 2026 for 2026–2028 pays $2,500 for these two young children at low taxable income."
+us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used a stale $7,830 EITC maximum and a $25,511 phase-out start, which is not the MFJ threshold, and phased the federal EITC down to $6,716. For 2026 MFJ filers, $30.5k of earned income still earns the full $8,231 (NJ EITC $3,292.40). It also used $500 per young child instead of the taxable-income-based CTC and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It overstated the federal EITC at $8,000–$9,000, giving about $3,800 of NJ EITC, but the 3+ child maximum is $8,231, so the NJ EITC is $3,292.40. It used the old $2,000 CTC and left out the $50 property tax credit, so its total is close to the reference only because these errors offset."
+us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied the $800 NJ CTC tier for $30,001–$40,000 using gross income. The NJ CTC is keyed to NJ taxable income, which is well under $30,000 after the personal, dependent and blind/disabled exemptions, and the correct amount is $2,500 for the two children under 6. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a federal EITC of $8,021 instead of the $8,231 maximum. It left out the refundable NJ child tax credit ($2,500 for the two children age 5 and under) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $7,787 federal EITC instead of the full $8,231. It omitted the refundable NJ child tax credit ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a federal EITC of $8,190 instead of the $8,231 2026 maximum and left out the $50 NJ property tax credit for this renter household. It also used the superseded $1,000-per-child CTC instead of the June 2026 schedule's $2,500 for two young children."
+us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a federal EITC of $8,247 instead of $8,231, and a $75 'renters' credit' instead of the $50 NJ property tax credit. It also used the old $2,000 CTC instead of the $2,500 that the June 30, 2026 schedule pays for two children under 6."
+us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used an understated $7,691 federal EITC instead of the unreduced $8,231. It put the NJ CTC in the $800 tier for $30,000–$40,000 based on gross income, although NJ taxable income is under $30,000 and the credit is $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no NJ refundable credits, missing the 40% NJ EITC ($3,292.40), the NJ child tax credit for two children under 6 ($2,500), and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It took the $800-per-child CTC tier from gross income near $30,900. NJ taxable income after exemptions is far below $30,000, which yields $2,500 for the two children under 6. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It phased the federal EITC down to $7,128, but $30.5k of MFJ earned income is below the 2026 phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It also used the $800 gross-income CTC tier instead of the taxable-income-based $2,500, and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no computation, and $1,200 is less than half of the NJ EITC alone. The correct total is 40% × $8,231 = $3,292.40 EITC, plus $2,500 CTC for two children under 6, plus the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its NJ EITC of $3,219.60 is based on the 2025 federal maximum of about $8,046 instead of the 2026 maximum of $8,231. Its $1,600 CTC applies the $800 gross-income tier instead of the NJ taxable-income test that yields $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"$4,776 equals a $1,600 CTC ($800 tier, chosen from gross income) plus about $3,176 of NJ EITC from an understated federal EITC. The correct components are $3,292.40 EITC, $2,500 CTC based on NJ taxable income under $30,000, and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right. It put the NJ CTC in the $800 tier because gross income exceeds $30,000, but the tier is set by NJ taxable income, which is well below $30,000 after exemptions, and the credit for the two young children is $2,500."
+us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so no substantive answer was given."
+us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used an overstated $8,360 federal EITC and counted only one child under 6, missing that the 5-year-old also qualifies. It also gave $500 instead of the $2,500 total the CTC pays for two children at low NJ taxable income, and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It found no refundable NJ credit. A married family of eight with about $30.5k of earned income receives the 40% NJ EITC ($3,292.40), the NJ CTC for two children under 6 ($2,500), and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It set refundable credits to zero because of 'no specific circumstances', but low-income status with qualifying children is exactly what triggers the NJ EITC ($3,292.40) and NJ CTC ($2,500), and rent triggers the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The $3,292.40 EITC is right, but the model left out the $50 NJ property tax credit even though $6,949 in rent is listed. It also applied the superseded $1,000-per-child CTC instead of the June 30, 2026 schedule, which pays $2,500 for the two children age 5 and under."
+us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It based the NJ EITC on the 2025 federal maximum of $8,046 instead of the 2026 maximum of $8,231. It left out the refundable NJ child tax credit ($2,500 for two children under 6) and the $50 property tax credit entirely."
+us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It gave $1,600 of CTC by using the $800 tier for incomes above $30,000, but the NJ CTC is keyed to NJ taxable income, which is well under $30,000 after exemptions, and the credit is $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 is the $3,292.40 EITC plus a $1,600 CTC from the $800 gross-income tier. The CTC is based on NJ taxable income below $30,000 and equals $2,500 for the two young children, and the model also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right, but it used the $800-per-child CTC tier for $30,001–$40,000. NJ taxable income after exemptions is far below $30,000, so the CTC for the two children under 6 is $2,500."
+us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied a 50% NJ EITC rate, but New Jersey's rate is 40% of the federal credit, giving $3,292.40 on the $8,231 federal EITC. It also used the old $2,000 CTC instead of $2,500 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It overstated the NJ EITC at $3,337 instead of 40% × $8,231 = $3,292.40, and used the $800-per-child CTC tier from gross income instead of the taxable-income-based $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It computed the $3,292.40 EITC and the $50 property tax credit correctly, but used the pre-2026 $1,000-per-child CTC ($2,000). The June 30, 2026 NJ schedule for 2026–2028 pays $2,500 for these two children under 6 at low taxable income."
+us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It said no state refundable credits apply. The household receives the refundable NJ EITC ($3,292.40), the NJ CTC ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It placed the NJ CTC in a $30,001–$40,000 bracket using gross income and used only $400 per child. The credit depends on NJ taxable income, which is under $30,000, and equals $2,500 for the two young children. It also overstated the federal EITC at $8,247 instead of $8,231 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $500-per-child NJ CTC instead of the $2,500 the credit pays for the two children under 6 at low NJ taxable income. It also overstated the federal EITC at $8,250 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It explicitly excluded the $50 NJ property tax credit because it saw 'no inferred rent', but the spouse's $6,949 rent is listed. It also used the superseded $1,000-per-child CTC instead of the June 30, 2026 schedule's $2,500 for the two young children."
+us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $7,901 federal EITC instead of $8,231, and stated there were no other credits. It missed the refundable NJ CTC ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an $8,271 federal EITC instead of $8,231. It omitted the refundable NJ child tax credit for the two children under 6 ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so no substantive answer was given."
+us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The $3,292.40 EITC is right, but the model left out the $50 NJ property tax credit that applies to this renter household. It also used the pre-2026 $1,000-per-child CTC instead of the June 30, 2026 schedule's $2,500 for two children under 6."
+us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no NJ refundable credits apply, missing the 40% NJ EITC ($3,292.40), the NJ CTC ($2,500) and the $50 property tax credit, all of which this family qualifies for."
+us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It phased down the NJ CTC because $30,915 of gross income exceeds $30,000. The CTC depends on NJ taxable income, which is far below $30,000 after exemptions, so no reduction applies and the credit is $2,500 under the 2026 schedule. It also overstated the federal EITC at $8,262 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It claimed the federal EITC phases out to near $0 at $30,800 of earned income. For MFJ filers with 3+ children, that income is still at the $8,231 maximum, so the NJ EITC is $3,292.40. It also wrongly zeroed the NJ CTC ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It wrongly said the household is above an NJ EITC phase-out and invented a $100 minimum credit. The NJ EITC is simply 40% of the full $8,231 federal EITC ($3,292.40), and the NJ CTC ($2,500) and $50 property tax credit are also due."
us,scenario_008,tanf,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model submitted a bare number with no derivation; the correct derivation applies New Jersey's WFNJ eligibility standard and yields $0, because countable income of about $2,576/month exceeds the family-of-eight standard (roughly $770/month) even after the 50% earned-income disregard, and $69,500 in bank assets exceeds the $2,000 TANF resource limit. Its $5,800 figure (about $483/month) is consistent with pulling a large-family NJ grant-schedule amount and shading it down for earnings — treating TANF as a residual gap-filler for a household with six children — while never running the income eligibility gate or the asset test at all."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got every step right and reached exactly $8,348: AGI $105,900, standard deduction $32,200, taxable income $73,700, 10% to $24,800 and 12% above that. It then submitted $3,455, a number that no step of its own computation supports."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted all $30,000 of Social Security instead of the $25,500 taxable portion. It also subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages and used the pre-OBBBA $30,000 standard deduction instead of $32,200. It then submitted $2,950, which does not match its own tax estimate of about $8,600."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly derived AGI of $105,900 and the 2026 brackets, but used a $32,300 standard deduction instead of $32,200, giving $8,336 (versus $8,348). It then submitted $4,880, which has no connection to its own $8,336 computation."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It used a $32,300 standard deduction (instead of $32,200) and a 10% bracket ending at $24,150 (instead of $24,800), and still reached $8,349. It then submitted $5,732, which no step of its reasoning produces."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It overstated the 2026 MFJ standard deduction as $32,600 instead of $32,200, giving taxable income of $73,300 and tax of $8,300. It then submitted $4,989, which is disconnected from its own $8,300 result."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 $30,000 standard deduction to $30,700 instead of using the 2026 MFJ amount of $32,200. It also projected the 10% bracket to end at $24,450 instead of $24,800. That overstated taxable income ($75,200 versus $73,700) and gave $8,535."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at $16,150 instead of the $25,500 cap (85% of $30,000), which understated AGI at $96,550. It also used a $32,600 standard deduction instead of $32,200. It then cut its own $7,178 to $6,350 for 'credit adjustments', even though this childless household has no nonrefundable credits."
-us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"AGI of $105,900 was correct, but it used a $30,750 standard deduction instead of the 2026 MFJ $32,200. It also ended the 10% bracket at $24,500 instead of $24,800. That overstated taxable income at $75,150 and gave $8,528."
-us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages, which cut AGI to $97,761 instead of $105,900. It also used a $30,600 standard deduction instead of $32,200 and a $24,400 top for the 10% bracket instead of $24,800."
-us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA individual provisions expired in 2026 and applied a $16,600 pre-TCJA standard deduction, $10,600 of personal exemptions and a 15% bracket. The OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no exemptions and a 12% bracket up to $100,800."
-us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules for 2026: a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket. The correct rules are the permanent TCJA/OBBBA $32,200 MFJ standard deduction and the 12% bracket, which give $8,348 on $73,700 of taxable income."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 ESI premium from wages and invented $14,640 of mortgage interest and $1,865.76 of state tax to itemize, even though no interest or tax paid was listed. Its $2,856.92 medical deduction is also wrong, because the medical costs are far below the 7.5%-of-AGI floor. On top of that it assumed TCJA had expired, adding personal exemptions and a 15% bracket instead of the $32,200 standard deduction and the 12% bracket."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $30,000 of Social Security instead of the $25,500 taxable portion and used a $10,040 standard deduction instead of $32,200. Its arithmetic was also wrong: the listed items net to $100,360, not the $80,360 it reported."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the spouse's $8,139 ESI premium from gross wages, putting AGI at $97,761 instead of $105,900. It also assumed TCJA had expired, using about $27,600 of standard deduction plus personal exemptions and a 15% bracket. The correct 2026 figures are the $32,200 MFJ standard deduction and the 10%/12% brackets."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reduced wages by the spouse's $8,139 ESI premium (AGI $97,761 instead of $105,900). It also applied a reverted pre-TCJA $16,256 standard deduction, $10,368 of personal exemptions and a 15% bracket, instead of the permanent 2026 $32,200 standard deduction with no exemptions and a 12% bracket."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Under the 2026 MFJ brackets, its $6,715 corresponds to taxable income of about $60,100, roughly $13,600 below the correct $73,700 ($105,900 AGI including $25,500 of taxable Social Security, minus the $32,200 standard deduction), so it understated taxable income."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $8,139 ESI premium as a pre-tax exclusion from the $60,000 gross wages, which understated AGI at $97,761. Its $7,558.32 is also higher than the $7,371.32 that its own AGI gives with the $32,200 deduction, so it also used a standard deduction smaller than $32,200."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"AGI of $105,900 was correct, but it subtracted personal exemptions and a pre-TCJA standard deduction as if TCJA had expired in 2026. The correct rule is the $32,200 MFJ standard deduction with no exemptions, taxed at 10%/12%, which gives $8,348."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 figures: a $30,000 standard deduction and a 10% bracket ending at $23,850. The 2026 figures are a $32,200 standard deduction and a 10% bracket to $24,800. That overstated taxable income at $75,900 and gave $8,631."
-us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,750 instead of $32,200 and ended the 10% bracket at $24,450 instead of $24,800. Taxable income came out at $75,150 instead of $73,700, giving $8,529."
-us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 MFJ standard deduction at $33,300 instead of $32,200 and used the 2025 10% bracket top of $23,850 instead of $24,800. That gave taxable income of $72,600 and tax of $8,235."
-us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the household falls below the taxable threshold without computing anything. $80,400 of wages and pension plus $25,500 of taxable Social Security gives $105,900 of AGI, which leaves $73,700 taxable after the $32,200 standard deduction and produces $8,348 of tax."
-us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It cited reductions from itemized health deductions. But about $1,600 of qualifying medical costs is far below the 7.5%-of-AGI floor ($7,943), and itemized deductions of $3,208 lose to the $32,200 standard deduction. Its $2,757 implies taxable income of only about $27,100, instead of $73,700."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It stated $26,150 of tax before credits and then submitted $3,922. Neither figure follows from $73,700 of taxable income ($105,900 AGI minus the $32,200 standard deduction) at 10% to $24,800 and 12% above, which gives $8,348."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA sunset and used a $17,018 standard deduction, $10,854 of personal exemptions and a 15% bracket. The OBBBA made the TCJA rules permanent, so the correct inputs are the $32,200 standard deduction, no exemptions and 12% up to $100,800."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law for 2026: a $16,550 standard deduction, $10,600 of personal exemptions and a 15% bracket. It should have used the permanent 2026 MFJ standard deduction of $32,200 and the 10%/12% brackets, which put taxable income at $73,700 instead of $78,750."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed TCJA had expired and used a $16,600 standard deduction, $10,600 of personal exemptions and a 15% bracket. The OBBBA extended the TCJA structure permanently, so 2026 has a $32,200 MFJ standard deduction, no exemptions and a 12% bracket up to $100,800."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA rules for 2026: a $15,700 standard deduction, $10,100 of exemptions and a 15% bracket. It should have used the permanent $32,200 MFJ standard deduction with no exemptions and the 10%/12% brackets, which give $8,348 on $73,700 of taxable income."
-us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly used the $32,200 standard deduction and the 10%/12% brackets. However, it subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages, which understated AGI at $97,761 instead of $105,900 and gave $7,371."
-us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction instead of the 2026 MFJ $32,200 and ended the 10% bracket at $24,400 instead of $24,800. That put taxable income at $75,300 instead of $73,700 and tax at $8,548."
-us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a $30,900 standard deduction instead of $32,200 and ended the 10% bracket at $24,500 instead of $24,800. Taxable income came out at $75,000 instead of $73,700, giving $8,510."
-us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a projected $30,450 standard deduction instead of $32,200 and the 2025 10% bracket top of $23,850 instead of $24,800, reaching $8,577. It then submitted $8,077.50, which drops $500 from its own total."
-us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at $4,000 instead of the $25,500 cap (85% of $30,000), which understated AGI at $84,440. It also invented a $1,050 child and dependent care credit, even though the household has no dependents or care expenses. Its final $12,110.90 is higher than its own $5,930.25 tax, so it did not subtract that credit."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got exactly the right result: AGI $105,900, minus the $32,200 standard deduction, gives $73,700, taxed at $2,480 + $5,868 = $8,348. It then submitted 3,455, a number that doesn't follow from any of its own work, so the error is an output mismatch rather than a tax-rule error."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,139 ESI premium from wages that are already gross W-2 income. It also counted all $30,000 of Social Security rather than the $25,500 taxable under the 85% cap, and used the 2025 $30,000 standard deduction instead of the 2026 $32,200. Its own estimate was about $8,600, yet it submitted $2,950 with no derivation."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It guessed taxable Social Security at about $13,000. With provisional income of $95,400 well above the $44,000 MFJ threshold, the 85% cap of $25,500 applies. It also reported $2,300 of tax on roughly $70,000 of taxable income, which contradicts the 10%/12% brackets: $73,700 of taxable income is taxed at $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It got AGI right at $105,900 and computed $8,336 using a $32,300 standard deduction, which should have been $32,200. It then submitted 4,880, a value disconnected from its own calculation."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed $8,349 from AGI of $105,900, though with a $32,300 standard deduction and a $24,150 bracket where the 2026 figures are $32,200 and $24,800. It then drifted to 'roughly $8,000' and submitted 5,732, which doesn't follow from any step it showed."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It used the correct 2026 brackets (10% to $24,800) and AGI of $105,900. It overstated the standard deduction at $32,600 instead of $32,200, which gave about $8,300, and then submitted 4,989, a number unrelated to its own computation."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the 2025 $30,000 standard deduction to $30,700 and used a $24,450 10% bracket. Under OBBBA the 2026 MFJ figures are $32,200 and $24,800. That overstated taxable income at $75,200 instead of $73,700, giving $8,535."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It said taxable Social Security was '85% inclusion' but put it at about $16,150, when 85% of $30,000 is $25,500. That understated AGI at $96,550 instead of $105,900. It also used a $32,600 standard deduction instead of $32,200, then cut the result further for 'credit adjustments' even though no nonrefundable credits apply."
+us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but used inflation-projected pre-OBBBA parameters: a $30,750 standard deduction and a $24,500 10% bracket. The OBBBA 2026 MFJ figures are $32,200 and $24,800, so its taxable income was $75,150 instead of $73,700 and its tax $8,528."
+us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,139 ESI premium from wages already given as gross income, which lowered AGI to $97,761 instead of $105,900. It also used a $30,600 standard deduction and a $24,400 bracket where the 2026 figures are $32,200 and $24,800."
+us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA expired for 2026 and used a $16,600 standard deduction, $10,600 of personal exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets that tax $73,700 at $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used pre-TCJA sunset rules: a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket. OBBBA kept the TCJA framework for 2026, with a $32,200 standard deduction and the 12% rate on income above $24,800."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired, applying personal exemptions and a 15% bracket. It also subtracted the ESI premium from gross wages and invented $14,640 of mortgage interest even though no interest amount was given. Actual itemized deductions are $3,208, below the $32,200 OBBBA standard deduction, which applies to $105,900 of AGI."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It skipped the Social Security taxability formula and counted the full $30,000 instead of the $25,500 cap. It used a $10,040 deduction unrelated to the 2026 $32,200 MFJ standard deduction, and its own arithmetic is wrong ($110,400 − $10,040 is not $80,360)."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired and used about $27,600 of pre-TCJA deductions and exemptions with a 15% bracket. It also subtracted the $8,139 ESI premium from gross wages, cutting AGI to $97,761 instead of $105,900. Under OBBBA the $32,200 standard deduction and 10%/12% brackets apply."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used reverted pre-TCJA 2026 parameters: a $16,256 standard deduction, $10,368 of exemptions and a 15% bracket above $23,872. It also subtracted the ESI premium from wages that are already gross. OBBBA made the $32,200 MFJ standard deduction and 12% bracket permanent, and AGI is $105,900."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is AGI $105,900 minus the $32,200 standard deduction, giving $73,700 and $8,348 of tax. Its $6,715 matches understating taxable income by about $13,600 in the 12% bracket, for example by under-including taxable Social Security or wrongly deducting premiums."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the $8,139 ESI premium as a pre-tax reduction of wages already reported as gross $60,000, which put AGI at $97,761 instead of $105,900. Its $7,558.32 also implies a standard deduction below the 2026 MFJ $32,200."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but applied personal exemptions and pre-TCJA 2026 brackets. OBBBA eliminated exemptions permanently and set a $32,200 MFJ standard deduction with a 12% bracket above $24,800, giving $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used 2025 parameters for 2026: a $30,000 standard deduction and a $23,850 10% bracket. The 2026 MFJ figures are $32,200 and $24,800, so its taxable income was $75,900 instead of $73,700 and its tax $8,631."
+us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but used an inflation-projected $30,750 standard deduction and a $24,450 bracket. The 2026 OBBBA figures are $32,200 and $24,800, so it overstated taxable income at $75,150 and produced $8,529."
+us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 MFJ standard deduction at $33,300 instead of $32,200 and used the 2025 10% bracket top of $23,850 instead of $24,800. Taxable income came out at $72,600 and tax at $8,235."
+us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the household's income was below the taxable threshold. In fact AGI of $105,900 (including $25,500 of taxable Social Security) exceeds the $32,200 standard deduction by $73,700, which produces $8,348 of tax."
+us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced tax using itemized health deductions. Deductible medical costs of about $1,600 are far below the 7.5% of AGI floor ($7,943), and total itemized deductions of $3,208 are below the $32,200 standard deduction, so the standard deduction applies and tax is $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Its reasoning cites $26,150 of tax before credits but it submitted 3,922, so the two contradict each other. Neither number comes from the correct $73,700 of taxable income (AGI $105,900 minus the $32,200 standard deduction), which is taxed at $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but assumed TCJA expired, applying a $17,018 standard deduction, $10,854 of exemptions and a 15% bracket. OBBBA kept the $32,200 MFJ standard deduction and 10%/12% brackets for 2026."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used post-sunset 2026 rules: a $16,550 standard deduction, $10,600 of exemptions and a 15% bracket above $24,325. OBBBA made the TCJA structure permanent, so taxable income is $73,700 under the 12% rate."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed TCJA's individual provisions expired, using $10,600 of exemptions, a $16,600 standard deduction and a 15% bracket. OBBBA kept the TCJA framework, so the 2026 MFJ standard deduction is $32,200 and the rate above $24,800 is 12%."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA rules for 2026: a $15,700 standard deduction, $10,100 of exemptions and a 15% bracket. OBBBA made the TCJA parameters permanent, so 2026 has a $32,200 standard deduction and 10%/12% brackets."
+us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction but subtracted the $8,139 ESI premium from wages already given as gross, making AGI $97,761 instead of $105,900. That understated taxable income at $65,561 instead of $73,700."
+us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but used an estimated $30,600 standard deduction and a $24,400 bracket. The OBBBA 2026 MFJ figures are $32,200 and $24,800, so its taxable income was $75,300 and its tax $8,548."
+us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It got AGI right at $105,900 but used a $30,900 standard deduction and a $24,500 bracket instead of the 2026 figures of $32,200 and $24,800. Taxable income came out at $75,000 and tax at $8,510."
+us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a projected $30,450 standard deduction and the 2025 $23,850 bracket instead of the 2026 OBBBA figures of $32,200 and $24,800, which produced $8,577. It then submitted 8,077.50, which doesn't match even its own total."
+us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as $4,000 instead of the $25,500 cap. It also invented a $1,050 CDCC even though there are no dependents or care expenses, and added it to tax rather than subtracting. The final $12,110.90 contradicts its own $5,930.25 tax figure."
us,scenario_009,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_009,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model improperly inferred SSDI and disability-based Medicare eligibility from Social Security retirement income despite the instruction that unlisted statuses are false. At age 57 with no disability, ESRD, or ALS stated, the Head does not satisfy either the age pathway or an under-65 Medicare pathway."
us,scenario_009,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly concluded that the 57-year-old Head is not Medicare eligible, but it submitted value = 1, which denotes eligible. This is an answer-value inversion: its stated conclusion required value = 0."
us,scenario_009,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_009,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $8,139 employer-sponsored insurance premium from FICA wages. The full $60,000 is subject to the 6.2% employee Social Security and 1.45% employee Medicare taxes, producing $4,590."
-us,scenario_009,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as an employee pre-tax deduction that reduced FICA wages to $51,861. FICA applies to the full $60,000 wage amount here, yielding $4,590 at the combined employee rate of 7.65%."
-us,scenario_009,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced the payroll-tax base by the $8,139 employer-sponsored insurance premium. Applying 7.65% to the unreduced $60,000 of wages gives $4,590."
-us,scenario_009,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly classified the $8,139 employer-sponsored insurance premium as a pre-tax employee contribution excluded from FICA wages. The taxable wage base remains $60,000, so employee Social Security is $3,720 and employee Medicare is $870."
-us,scenario_009,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model wrongly deducted the employer-sponsored insurance premium from the FICA wage base. The full $60,000 is subject to the 7.65% combined employee FICA rate, resulting in $4,590."
-us,scenario_009,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model used the combined employer-and-employee rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side payroll taxes. It also reported $13,663, which does not follow from applying its stated 15.3% rate to the spouse's $60,000 of wages; the employee rates produce $4,590."
-us,scenario_009,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages to $51,861 by subtracting the employer-sponsored insurance premium. The employee Social Security and Medicare rates apply to the entire $60,000 wage amount, totaling $4,590."
+us,scenario_009,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model correctly computed $3,720 Social Security plus $870 Medicare = $4,590 on the spouse's $60,000 wages, and its explanation even ends with value = 4590. It then added a made-up $1,230 of 'SS/Medicare adjustments' and submitted $5,820. No extra employee payroll tax applies at this wage level, and NC has no employee state payroll tax."
+us,scenario_009,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the spouse's $8,139 employer-sponsored insurance premiums from wages as a pre-tax cafeteria-plan exclusion. That shrank the FICA base to $51,861. No salary-reduction arrangement is listed, so FICA applies to the full $60,000 gross wages: 7.65% × $60,000 = $4,590."
+us,scenario_009,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $8,139 ESI premium as a pre-tax payroll deduction and applied 7.65% to $51,861. The household facts give no Section 125 salary reduction, so Social Security and Medicare apply to the full $60,000 gross wages, giving $4,590."
+us,scenario_009,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model excluded the spouse's $8,139 ESI premiums from FICA wages, though no pre-tax payroll deduction is listed, and taxed only $51,861. The correct base is the full $60,000 gross wages: $3,720 Social Security + $870 Medicare = $4,590."
+us,scenario_009,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed the $8,139 ESI premium was withheld pre-tax and computed FICA on $51,861. FICA wages are the stated $60,000 gross wages, so the correct figures are $3,720 Social Security and $870 Medicare, totaling $4,590."
+us,scenario_009,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted the $8,139 employer-sponsored insurance premium as a FICA-exempt deduction and taxed $51,861. No cafeteria-plan salary reduction is given, so 7.65% applies to the full $60,000, giving $4,590."
+us,scenario_009,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model used the combined employer-plus-employee rates (12.4% Social Security and 2.9% Medicare) instead of the employee-only 6.2% and 1.45%. Its $13,663 is 15.3% of about $89,300, so it also taxed non-wage income such as pension or Social Security on top of the spouse's $60,000 wages. The correct employee-side figure is 7.65% × $60,000 = $4,590."
+us,scenario_009,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model reduced the spouse's FICA wages to $51,861 by subtracting the $8,139 ESI premiums as pre-tax, but no salary-reduction arrangement is listed. Employee Social Security and Medicare apply to the full $60,000 gross wages, giving $4,590."
us,scenario_009,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_009,spouse_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model correctly concluded that the 57-year-old spouse was not Medicare eligible but then submitted value = 1, reversing the prompt’s binary encoding. “Not eligible” required value = 0."
us,scenario_009,spouse_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It worked out the correct chain: AGI of $105,900, minus $25,500 Social Security, minus the $25,500 standard deduction, equals $54,900, and 3.99% of that is $2,190.51. It then submitted 2,726, a number that contradicts its own derivation and has no supporting step."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It never subtracted the taxable Social Security that North Carolina excludes, and it used a $25,000 standard deduction instead of $25,500. It then made up unnamed 'federal nonrefundable credits' that wipe out the North Carolina tax. North Carolina has no such credits for this childless couple, so the real liability is $54,900 × 3.99% = $2,190.51, not $0."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income. It then applied 4.25% (the 2025 rate) instead of the 2026 flat rate of 3.99%. It also submitted 2,891, which does not match its own 4.25% figure of $2,333."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed $54,900 × 3.99% = $2,191. It then invented an 'adjustment for taxable SS included then deducted' that raised the answer to 2,974. This counts the Social Security subtraction twice, even though that subtraction is already built into the $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It reached $54,900 of taxable income and $2,190 of tax, then threw that out. Instead it applied 3.99% to a made-up $71,600 base that skips the standard deduction, which gives $2,856."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It cut the spouse's wages by the $8,139 employer-sponsored insurance premium, which lowered AGI to $97,761; the reference keeps the full $60,000 of gross wages in AGI. It also assumed about $25,620 of mortgage interest from the loan balance and itemized with it, even though no mortgage interest was listed. That gave $46,641 × 3.99% = $1,861 instead of $54,900 × 3.99%."
-us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used 4.25% instead of the 2026 rate of 3.99%. It subtracted the full $30,000 of Social Security instead of the $25,500 actually included in federal AGI, and it floated an itemized deduction built on assumed mortgage interest. It then submitted 3,200, which none of its intermediate figures ($2,142, $1,912) support."
-us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $26,000 married-filing-jointly standard deduction instead of North Carolina's $25,500. That gives $54,400 × 3.99% = $2,171 instead of $54,900 × 3.99% = $2,190.51."
-us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $97,761, which takes the $8,139 employer insurance premium out of wages; the reference AGI is $105,900. It also used a $26,500 standard deduction instead of $25,500, which gave $45,761 of taxable income instead of $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction (close to the federal amount) instead of North Carolina's $25,500 for married filing jointly. It also used a 4.25% rate instead of the 2026 rate of 3.99%, which gives 4.25% × $50,400 instead of 3.99% × $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income but taxed it at 4.25% instead of the 2026 flat rate of 3.99%. That produces $2,333.25 instead of $2,190.51."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 of gross wages, which cut North Carolina income to $72,261 instead of $80,400. Its taxable income was therefore $46,761 instead of $54,900, even though its rate (3.99%) and standard deduction ($25,500) were right."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied a 4.5% rate instead of the 2026 rate of 3.99%. It also overstated taxable income at about $60,800 instead of $54,900 (AGI of $105,900 minus $25,500 Social Security minus the $25,500 standard deduction)."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a federal AGI of $97,761, which treats the spouse's $8,139 insurance premium as a pre-tax cut to wages. The reference AGI includes the full $60,000 of wages and comes to $105,900. That left taxable income at $46,761 instead of $54,900 under the correct 3.99% rate."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $97,761, which removes the $8,139 employer premium from wages. It also used a $29,250 standard deduction instead of North Carolina's $25,500 for married filing jointly, which gave $43,011 of taxable income instead of $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,212 matches 3.99% applied to about $80,400 (wages plus pension) with no $25,500 standard deduction. The correct taxable income is $54,900, which gives $2,190.51."
-us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took federal AGI as $97,761 by netting the spouse's $8,139 employer insurance premium out of wages. That understates North Carolina income by $8,139 and gives $46,761 of taxable income instead of $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed zero North Carolina tax without computing a taxable income. The couple has $80,400 of North Carolina income (wages plus pension, with Social Security excluded). After the $25,500 standard deduction that leaves $54,900, taxed at 3.99%."
-us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It estimated zero tax without any computation. It missed that taxable income is $54,900 after the Social Security subtraction and the $25,500 standard deduction, and that no nonrefundable credit offsets the 3.99% tax."
-us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied a 4.5% flat rate instead of the 2026 rate of 3.99%. Its $2,340 implies about $52,000 of taxable income instead of the correct $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly reached $80,400 of North Carolina income but used a $21,000 standard deduction instead of $25,500. That gives $59,400 × 3.99% = $2,370 instead of $54,900 × 3.99%."
-us,scenario_009,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its North Carolina base of $72,261 comes from subtracting the spouse's $8,139 employer insurance premium from wages. The reference base is $80,400 (the full $60,000 of wages plus the $20,400 pension), so its taxable income was $46,761 instead of $54,900."
-us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for state_income_tax_before_refundable_credits, so it missed the $2,190.51 result (3.99% of $54,900)."
-us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It never applied North Carolina's subtraction for the $25,500 of Social Security included in federal AGI. It also used a $26,200 standard deduction and a 4.0% rate. That taxed $79,700 instead of $54,900 at 3.99%."
-us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income. It then applied 4.5% instead of the scheduled 2026 flat rate of 3.99%, which gave $2,470.50 instead of $2,190.51."
-us,scenario_009,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It made up North Carolina credits and exclusions that do not exist for this household: a $6,000 retirement exclusion, a state earned income credit, a child tax credit for a childless couple, and a 'retirement exclusion credit'. It used them to drive the tax to $0. The only adjustments are the Social Security subtraction and the $25,500 standard deduction, which leave $54,900 taxed at 3.99%."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own derivation was correct: $105,900 AGI, less $25,500 Social Security and the $25,500 standard deduction, gives $54,900, and 3.99% of that is $2,190.51. It then submitted $2,726, a number its reasoning never supports."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It never subtracted the $25,500 of taxable Social Security, which NC exempts, and it used a $25,000 standard deduction instead of $25,500. It also claimed unnamed nonrefundable credits wipe out the tax. NC has no nonrefundable credit available to this childless couple, so the tax on $54,900 at 3.99% stands."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It treated the private pension as partially exempt. NC's Bailey exclusion covers only certain government pensions, so the $20,400 private pension is fully taxable. Its rough $2,100 implies a taxable base below the correct $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct $54,900 taxable income but applied 4.25%, the 2025 rate, instead of the scheduled 2026 rate of 3.99%. It then submitted $2,891, which matches neither of its own calculations ($2,333 or $2,471)."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed 3.99% × $54,900 = $2,191. It then invented an unexplained Social Security 'adjustment' that raised the answer to $2,974, even though the $25,500 Social Security subtraction was already reflected in the $54,900 base."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It first derived $54,900 and $2,190. It then applied 3.99% to an unsupported $71,600 base, which drops part of the $25,500 standard deduction, and submitted $2,856."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,139 ESI premium from the spouse's $60,000 wages, even though the listed premiums do not reduce gross wages. It also invented $25,620 of mortgage interest from the $366,000 balance; no interest was listed, so interest is $0 and the $25,500 standard deduction applies. Together these understated NC taxable income ($46,641 instead of $54,900)."
+us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a 4.25% rate instead of 3.99% and subtracted the full $30,000 of Social Security instead of the $25,500 in AGI. It also assumed mortgage interest and property taxes that were never provided, then submitted $3,200, which none of its intermediate figures ($2,142 or $1,912) support."
+us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $26,000 MFJ standard deduction. NC's MFJ standard deduction is fixed by statute at $25,500 and is not inflation-indexed, so it understated taxable income by $500 ($54,400 instead of $54,900)."
+us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut federal AGI to $97,761 by subtracting the $8,139 ESI premium from wages, which do not take that reduction. It also used a $26,500 standard deduction instead of NC's fixed $25,500."
+us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a 4.25% rate instead of the 2026 rate of 3.99%. It also used a $30,000 NC standard deduction, the federal figure, instead of NC's $25,500 MFJ deduction."
+us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It derived the correct $54,900 taxable income but applied 4.25%, the 2025 rate, instead of the 2026 flat rate of 3.99%."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $8,139 ESI premium as a pre-tax reduction to the $60,000 wages, but the listed premiums do not reduce wages. That lowered taxable income to $46,761 instead of $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied an outdated 4.5% rate instead of 3.99% to a taxable income of about $60,800. That figure does not match the correct $54,900 (AGI $105,900, less $25,500 Social Security and the $25,500 standard deduction)."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $97,761 AGI subtracts the spouse's $8,139 ESI premium from wages, but the listed premiums do not reduce gross wages. That understated NC taxable income at $46,761 instead of $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a $97,761 AGI that wrongly subtracts the $8,139 ESI premium from wages. It also used a $29,250 standard deduction instead of NC's fixed $25,500 MFJ amount."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,212 is consistent with 3.99% applied to about $80,500, roughly AGI after the Social Security subtraction, which means it skipped the $25,500 MFJ standard deduction."
+us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $97,761 federal AGI comes from subtracting the $8,139 ESI premium from the spouse's wages, which is not allowed here. That produced $46,761 taxable income instead of $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It defaulted to $0 without computing anything. After the Social Security subtraction and the $25,500 standard deduction, NC taxable income is $54,900, which carries $2,190.51 of tax at 3.99%."
+us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed deductions and credits eliminate the tax without computing them. NC taxable income is $54,900 after the Social Security subtraction and standard deduction, and no NC nonrefundable credit applies, so the 3.99% tax is $2,190.51."
+us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied an outdated 4.5% flat rate instead of the 2026 rate of 3.99%. Its $2,340 implies a taxable base of about $52,000 instead of $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $21,000 MFJ standard deduction. NC's MFJ standard deduction has been $25,500 since 2023, so it overstated taxable income at $59,400 instead of $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its NC base of $72,261 implies it subtracted the spouse's $8,139 ESI premium from wages. Gross wages of $60,000 are not reduced by that premium, so the correct base is $80,400 and taxable income is $54,900."
+us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was no substantive answer to score."
+us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It never applied NC's subtraction for the $25,500 of Social Security included in federal AGI. It also used a 4.0% rate instead of 3.99% and a $26,200 standard deduction instead of NC's fixed $25,500."
+us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It derived the correct $54,900 NC taxable income but applied a 4.5% rate. The 2026 NC flat rate is 3.99%."
+us,scenario_009,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It invented a $6,000 retirement exclusion, a state earned income tax credit, a state child tax credit and a 'retirement exclusion credit'. NC offers none of these to this childless couple with a private pension. It also missed the Social Security subtraction and used a 4.5% rate."
us,scenario_009,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_009,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_012,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and income limits and omitted the requirement that a CHIP child be ineligible for Medicaid. Child 1 qualifies for Medicaid under the OLDER_CHILD category, which precludes CHIP eligibility regardless of being below the CHIP income ceiling."
-us,scenario_012,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below Mississippi's CHIP ceiling and age under 19 as sufficient eligibility conditions. It failed to test the child's Medicaid eligibility under the OLDER_CHILD category, which makes the child ineligible for CHIP."
-us,scenario_012,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model concluded that low income relative to the CHIP limit directly establishes CHIP eligibility. It omitted Medicaid screening: the 10-year-old is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP.
-us,scenario_012,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age, income, and absence of listed employer coverage but never applied CHIP's Medicaid-ineligibility condition. Child 1 qualifies for Medicaid under the OLDER_CHILD category, so the CHIP income comparison does not produce CHIP eligibility."
-us,scenario_012,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model treated household income below the separate CHIP program's ceiling as sufficient for eligibility. It omitted that Medicaid eligibility takes precedence: Child 1 qualifies under the OLDER_CHILD Medicaid category and is consequently excluded from CHIP.
-us,scenario_012,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not Medicaid-covered—but failed to evaluate it before returning 1. The Medicaid test places Child 1 in the OLDER_CHILD category, so that condition fails and CHIP eligibility is 0."
-us,scenario_012,child1_chip_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model relied exclusively on age and income being within Mississippi's CHIP limits. It skipped the prior Medicaid determination, which finds Child 1 eligible under the OLDER_CHILD category and thereby bars CHIP eligibility."
-us,scenario_012,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model expressly asserted that Child 1 was not covered by Medicaid without applying the child's own Medicaid eligibility rules. Child 1 independently qualifies for Medicaid under the OLDER_CHILD category, and that eligibility precludes CHIP."
-us,scenario_012,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly concluded that $22,000 of household wages exceeds the applicable Medicaid threshold for a 10-year-old in Mississippi. The correct Medicaid computation qualifies Child 1 under the OLDER_CHILD category, which in turn makes the child ineligible for CHIP."
+us,scenario_012,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared income only to Mississippi's roughly 211% CHIP ceiling and never checked whether the child qualifies for Medicaid first. It also miscalculated income as about 145% FPL. The correct figure is about 68% FPL for a family of 4, well under the 133%+5% Medicaid limit for older children. That makes the child Medicaid-eligible and therefore ineligible for CHIP."
+us,scenario_012,child1_chip_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,The model concluded CHIP eligibility from 'low income' alone. It did not recognize that a 10-year-old at about 83% FPL qualifies for Mississippi Medicaid in the older-child category. Medicaid eligibility excludes CHIP eligibility.
+us,scenario_012,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model tested income only against the 209% FPL CHIP ceiling and treated 'well below' that limit as qualifying. It missed that income this low also falls under the Medicaid limit for children aged 6-18 (133% FPL plus 5% disregard). The child is therefore placed in Medicaid, and CHIP covers only children who are not Medicaid-eligible."
+us,scenario_012,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model reasoned that income far below the roughly 209% CHIP limit makes the child CHIP-eligible. It never applied the lower Medicaid test for older children, which this child meets at about 83% FPL. Medicaid eligibility takes priority and makes the child ineligible for CHIP."
+us,scenario_012,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked income only against an assumed 200% FPL CHIP limit and the absence of employer coverage. It skipped the Medicaid screen: at about 83% FPL, the 10-year-old meets Mississippi's 133%+5% older-child Medicaid standard. Because CHIP excludes Medicaid-eligible children, the answer is 0."
+us,scenario_012,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated income below the roughly 214% CHIP threshold as enough for CHIP. It ignored that income at about 68-83% FPL qualifies the child for Medicaid in the older-child category, and CHIP is only available to children who do not qualify for Medicaid."
+us,scenario_012,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model correctly stated that CHIP applies only if the child is not Medicaid-eligible, but it assumed the child is not Medicaid-eligible. At $22,000 (about 83% FPL for 3), the 10-year-old is under Mississippi's 133%+5% older-child Medicaid limit. That makes the child Medicaid-eligible and therefore CHIP-ineligible."
+us,scenario_012,child1_chip_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model compared income only to the roughly 209% FPL CHIP ceiling. It never checked the Medicaid older-child limit (133% FPL plus 5% disregard), which this household's income of about 83% FPL easily meets. Medicaid eligibility takes priority over CHIP."
+us,scenario_012,child1_chip_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model put $22,000 at about 137% FPL for a family of 3. The actual figure is about 83% using the $26,650 FPL, so it overstated the income ratio. It also asserted the child is not Medicaid-eligible, but income this low clears Mississippi's 133%+5% Medicaid limit for children aged 6-18. That puts the child in Medicaid and rules out CHIP."
+us,scenario_012,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model claimed household income exceeds Medicaid child thresholds, which is false. $22,000 is about 83% FPL for a family of 3, well under Mississippi's 133% FPL (plus 5% disregard) Medicaid limit for 10-year-olds. The child is therefore Medicaid-eligible in the OLDER_CHILD category, which excludes CHIP."
+us,scenario_012,child1_head_start_eligible,claude-haiku-5.5,llm_error,other,False,"The model's explanation applies the Head Start preschool age limit correctly, concludes that the 10-year-old is not eligible, and ends with 'value = 0'. Its recorded answer, however, is Yes (1). The submitted numeric value contradicts the explanation, which violates the output contract requiring the value to match the explanation's stated conclusion."
us,scenario_012,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to place the 10-year-old dependent in Mississippi’s OLDER_CHILD Medicaid category and therefore never applied its MAGI income threshold. At 0.81 times FPL, the child passes that category’s income test and is eligible."
-us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 10 as satisfying WIC's child category. WIC covers children under age five, so this child is categorically ineligible regardless of the household's low income."
-us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the WIC income limit without first enforcing the program's categorical age restriction. A 10-year-old is outside WIC's under-five child category, so income below the limit does not confer eligibility."
-us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model explicitly misidentified age 10 as within WIC's age limit. WIC child eligibility ends at age five, making this 10-year-old categorically ineligible even though household income satisfies the financial threshold."
+us,scenario_012,child1_wic_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model correctly said that a 10-year-old is over WIC's under-5 age limit and is not eligible, but it still submitted value = 1. Its own reasoning required an answer of 0, so the number it submitted contradicts its explanation."
+us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated a 10-year-old as eligible under WIC's age-based child category. That category covers only children under 5 and ends at the 5th birthday, so Child 1 is categorically ineligible whatever the household income."
+us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,age_disability,False,"The model based eligibility only on household income being under the WIC limit and skipped the categorical requirement. A child qualifies for WIC only before the 5th birthday, so a 10-year-old is ineligible."
+us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model wrongly said that age 10 is within WIC's age limit. WIC child eligibility ends at the 5th birthday, so Child 1 fails the categorical test even though household income is below the income limit (185% of the federal poverty line)."
us,scenario_012,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived EITC $4,427 + ACTC $1,700 = $6,127. It then added an unexplained $966, reaching $7,093, even after stating that no other refundable credits apply. The extra amount has no basis in any credit this household qualifies for."
-us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model left out the refundable CTC entirely, claiming it was limited by an income phase-out. The CTC phase-out starts at $400,000 for MFJ, so the full $1,700 ACTC applies at $22,000. It also counted the pregnant spouse as a qualifying child and used head-of-household status. It then plugged in a stale $3,995 EITC maximum instead of the 2026 one-child maximum of $4,427."
-us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model got ACTC right at $1,700 but started from the 2025 one-child EITC maximum of $4,328. It then applied an arbitrary 'inflation adjustment' to $6,172, which implies an EITC of $4,472 instead of the 2026 maximum of $4,427."
-us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed ACTC at $1,700 correctly and first estimated the EITC near $4,400. It then replaced that with a 'refined' 2026 one-child maximum of $4,731, which overstates the actual $4,427 maximum by $304."
-us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an $1,800 per-child refundable CTC cap instead of the 2026 cap of $1,700. It also put the EITC at $4,367 by placing $22,000 near the phase-out. At $22,000 the family is on the plateau and gets the full $4,427 maximum."
-us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA $2,000 CTC and treated all of it as refundable. It never applied the $1,700 ACTC refundable cap. For the EITC it fell back on the 2025 one-child maximum of $4,328 instead of the 2026 maximum of $4,427."
-us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used stale one-child EITC parameters ($11,610 phase-in end, $3,995 plateau) and wrongly put $22,000 in the phase-out range, reaching about $3,200. It then dropped its own work and substituted an unsupported $5,500 EITC. At $22,000 the full 2026 one-child maximum of $4,427 applies, plus $1,700 ACTC."
-us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model had the structure right: full one-child EITC plus $1,700 ACTC. But it used $4,404 as the 2026 one-child EITC maximum instead of $4,427, which leaves the total $23 short."
-us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset ACTC rules: a $1,000 cap and a $3,000 earnings threshold. OBBBA made the $2,200 CTC permanent with a $1,700 refundable cap and a $2,500 threshold. It also understated the 2026 one-child EITC maximum as $4,355 instead of $4,427."
-us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model capped the refundable CTC at the pre-TCJA $1,000 instead of the 2026 ACTC cap of $1,700, which alone loses $700. It also put the 2026 one-child EITC maximum at $4,442 instead of $4,427."
-us,scenario_012,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model set the ACTC at $1,749, which exceeds the 2026 per-child refundable cap of $1,700. It also overstated the one-child EITC maximum as $4,454 instead of $4,427."
-us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC, the TCJA-sunset figure. The 2026 ACTC cap under OBBBA is $1,700, and 15% of earnings above $2,500 ($2,925) exceeds it. Its EITC of $4,423 is also slightly below the $4,427 one-child maximum."
-us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model overstated the EITC at $5,571, well above the 2026 one-child maximum of $4,427. It also understated the refundable CTC at the sunset-law $1,000 instead of the $1,700 ACTC cap."
-us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC instead of the 2026 ACTC cap of $1,700. It also rounded the EITC to $4,500 instead of the exact one-child maximum of $4,427."
-us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $1,000 for the refundable CTC instead of the 2026 ACTC cap of $1,700. It also used an EITC of $4,320, below even the 2025 one-child maximum, instead of the 2026 maximum of $4,427."
-us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no computation. The correct derivation is the full one-child EITC of $4,427 (earnings are on the plateau) plus the $1,700 ACTC. Its $4,050 is below the EITC maximum alone, so it either left out the ACTC entirely or badly understated both credits."
-us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got ACTC right at $1,700 but used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427. That leaves the total $99 short."
-us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model got the $1,700 ACTC right but used the 2025 one-child EITC maximum of $4,328 instead of the 2026 maximum of $4,427. That understates the total by $99."
-us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 maximum of $4,427. It also used the TCJA-sunset $1,000 ACTC instead of the 2026 refundable cap of $1,700."
-us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model set the refundable CTC to $0 because there was no tax liability, which has refundability backwards. With zero liability the full $1,700 ACTC is paid out, since 15% × ($22,000 − $2,500) exceeds the cap. It also put the EITC at $2,591 even though $22,000 is on the one-child plateau, where the full $4,427 maximum applies."
-us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 CTC as refundable and never applied the 2026 per-child refundable cap of $1,700. It also used $4,420 for the EITC instead of the $4,427 one-child maximum."
-us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model denied the EITC because the spouse is 18. The age-25 minimum applies only to the childless EITC; a joint filer with a qualifying child faces no age test, so the full $4,427 applies. Its $2,524 refundable CTC also exceeds the 2026 per-child ACTC cap of $1,700."
-us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0, claiming filing status and earned income were not specified. The facts give $22,000 in wages, a married couple filing jointly (filing is assumed), and a 10-year-old qualifying child. Those facts yield EITC $4,427 plus ACTC $1,700."
-us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated the 2026 one-child EITC maximum as $4,526 instead of $4,427. It also used $1,800 for the refundable CTC, above the 2026 ACTC cap of $1,700."
-us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the correct 2026 one-child EITC maximum of $4,427. Its only error is putting the refundable CTC at $1,800 instead of the 2026 ACTC cap of $1,700, which overshoots by $100."
-us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model applied phase-out reductions and got a $2,479 EITC. But $22,000 is past the one-child phase-in end and below the MFJ phase-out start, so the full $4,427 maximum applies. It also left out the $1,700 ACTC entirely."
-us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the post-sunset ACTC: a $1,000 cap above $3,000 of earnings. OBBBA made the TCJA CTC permanent, so the 2026 refundable cap is $1,700 above $2,500. It also understated the one-child EITC maximum as $4,318 instead of $4,427."
-us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and capped the ACTC at $1,000. Under OBBBA the 2026 refundable CTC cap is $1,700. It also set the EITC at $4,436 instead of the 2026 one-child maximum of $4,427."
-us,scenario_012,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model's structure is correct: plateau EITC plus the $1,700 ACTC cap. But it used $4,437 as the 2026 one-child EITC maximum instead of $4,427, which overshoots by $10."
-us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied the pre-TCJA ACTC ($1,000 cap, 15% above $3,000) instead of the 2026 OBBBA refundable cap of $1,700. It also inflated the 2025 EITC to $4,436 instead of using the 2026 one-child maximum of $4,427."
-us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_refundable_credits, so there was no answer to score. The correct derivation is EITC $4,427 plus ACTC $1,700."
-us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied the ACTC correctly as min($1,700, 15% × earnings above $2,500) = $1,700. But it put the 2026 one-child EITC maximum at $4,401 instead of $4,427, leaving the total $26 short."
-us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model claimed the EITC and refundable CTC 'partially offset' each other and returned $0. Both credits are positive and additive for this household: the full one-child EITC of $4,427 on the plateau plus the $1,700 ACTC, since tax liability is zero."
-us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the plateau EITC and the $1,700 ACTC cap correctly. It used $4,425 for the 2026 one-child EITC maximum instead of $4,427, a $2 shortfall."
-us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model counted the unborn child as a second qualifying child. That doubled the ACTC to $3,400 and switched to the two-child EITC maximum of $7,830. An unborn child is not a qualifying child for either credit, so the correct amounts are the one-child EITC of $4,427 and a single $1,700 ACTC."
-us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an EITC of $6,936, a multi-child figure far above the 2026 one-child maximum of $4,427. It also claimed the CTC was used against regular tax. That is wrong: $22,000 is below the MFJ standard deduction, so tax liability is zero and $1,700 is paid as ACTC."
+us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly found EITC $4,427 + ACTC $1,700 = $6,127. It then added an unexplained $966 (it said the pregnancy does not add a qualifying child) and submitted $7,093. That is an arithmetic error added on top of the correct derivation."
+us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted the pregnant spouse/unborn child as qualifying children and claimed three, but only the 10-year-old qualifies for EITC or CTC. It also used a head-of-household figure of $3,995 instead of the $4,427 joint one-child maximum for 2026, and it left out the $1,700 refundable ACTC entirely."
+us,scenario_012,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model claimed the EITC is about $0 at this income, but $22,000 is on the one-child plateau and earns the full $4,427. It also said the refundable CTC does not apply, even though zero tax liability and $19,500 of earnings above $2,500 produce a $1,700 ACTC."
+us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427. It then added an arbitrary inflation adjustment to reach $6,172 instead of $4,427 + $1,700 = $6,127. The $1,700 ACTC was correct."
+us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model replaced its ~$4,400 EITC estimate with a 'refined' one-child maximum of $4,731. The actual 2026 maximum is $4,427, so the EITC was overstated by $304."
+us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model capped the ACTC at $1,800 per child, but the 2026 refundable cap is $1,700. It also said the EITC was near phase-out (~$4,367), when $22,000 is below the joint phase-out start and gets the full $4,427."
+us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model kept the 2025 EITC maximum of $4,328 instead of the 2026 value of $4,427. It also treated the whole $2,000 CTC as refundable, ignoring the $1,700 per-child refundable cap on the ACTC."
+us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used stale one-child figures ($3,995 plateau, $11,610 phase-in) and wrongly put $22,000 in the phase-out range. It then dropped that work and submitted an unsupported ~$5,500 EITC; the correct full 2026 one-child EITC is $4,427, which with the $1,700 ACTC totals $6,127."
+us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,404 as the 2026 one-child EITC maximum instead of the indexed $4,427. That understated the total by $23."
+us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the pre-TCJA sunset rules: a $1,000 CTC with a $3,000 earnings threshold. Under OBBBA, the CTC is permanent with a $1,700 refundable cap and a $2,500 threshold. The model also understated the 2026 one-child EITC as $4,355 instead of $4,427."
+us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model limited the ACTC to the post-sunset $1,000, but OBBBA's 2026 refundable cap is $1,700. It also overstated the one-child EITC maximum as $4,442 instead of $4,427."
+us,scenario_012,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model overstated both parts. It used an EITC of $4,454 instead of the 2026 one-child maximum of $4,427, and an ACTC of $1,749 when the 2026 refundable cap is exactly $1,700 per child."
+us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the expired $1,000 refundable CTC instead of OBBBA's 2026 refundable cap of $1,700. It also put the EITC at $4,423 instead of $4,427."
+us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model put the one-child EITC at ~$5,571, well above the 2026 one-child maximum of $4,427. It also used the sunset-era $1,000 refundable CTC instead of the $1,700 OBBBA refundable cap."
+us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model rounded the EITC to $4,500 instead of the exact 2026 one-child maximum of $4,427. It also used a $1,000 refundable CTC instead of the 2026 ACTC cap of $1,700."
+us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used ~$4,320 for the one-child EITC instead of the 2026 maximum of $4,427. It also used a post-sunset $1,000 refundable CTC instead of the $1,700 OBBBA refundable cap."
+us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $4,050 is below the one-child EITC maximum alone ($4,427), so it understated the EITC and dropped all or nearly all of the $1,700 refundable ACTC that brings the correct total to $6,127."
+us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427, which understated the total by $99."
+us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 indexed $4,427, which understated the total by $99."
+us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum of $4,328 instead of the 2026 value of $4,427. It also used a $1,000 ACTC from the TCJA sunset instead of the permanent OBBBA $1,700 refundable cap."
+us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model applied a reduction that cut the EITC to $2,591, even though $22,000 is past the phase-in end and below the joint phase-out start, so the full $4,427 applies. It also got refundability backwards: it said the ACTC is $0 because there is no tax liability, when zero liability is exactly what makes the full $1,700 refundable."
+us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 OBBBA CTC as refundable, but the 2026 refundable portion (ACTC) is capped at $1,700 per child. It also put the EITC at $4,420 instead of $4,427."
+us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model denied the EITC because the spouse is 18, but the 25–64 age test applies only to filers without qualifying children. A joint return with a qualifying 10-year-old gets the full $4,427 regardless of the spouse's age. Its $2,524 refundable CTC also exceeds the $1,700 per-child ACTC cap."
+us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed filing status and earned income were unspecified and returned $0, but the facts give $22,000 of wages, a married couple, and a 10-year-old child. That supports a joint one-child EITC of $4,427 plus a $1,700 ACTC."
+us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated both parts. It used $4,526 for the 2026 one-child EITC instead of $4,427, and an $1,800 ACTC instead of the $1,700 refundable cap."
+us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The EITC of $4,427 was correct, but the model put the refundable CTC at $1,800 instead of the 2026 per-child refundable cap of $1,700, which overstated the total by $100."
+us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model applied a phase-out that cut the EITC to $2,479, even though $22,000 is on the one-child plateau and gets the full $4,427. It also left out the $1,700 refundable ACTC."
+us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the post-sunset ACTC ($1,000 cap, $3,000 threshold), but OBBBA made the CTC permanent with a $1,700 refundable cap in 2026. It also used $4,318 for the EITC maximum instead of $4,427."
+us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the post-sunset $1,000 ACTC maximum instead of the OBBBA 2026 refundable cap of $1,700. It also overstated the one-child EITC as $4,436 instead of $4,427."
+us,scenario_012,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,437 as the 2026 one-child EITC maximum instead of $4,427, which overstated the total by $10."
+us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC (sunset rules, $3,000 threshold) instead of OBBBA's $1,700 2026 refundable cap. It also projected the EITC at $4,436 instead of the published $4,427."
+us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no reasoning for federal_refundable_credits, so there is no answer to score against the $6,127 EITC-plus-ACTC total."
+us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,401 as the 2026 one-child EITC maximum instead of $4,427, which understated the total by $26."
+us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model said the EITC and refundable CTC 'partially offset' and returned $0, but the two credits add together. $22,000 of earnings with one qualifying child gets the full $4,427 EITC plus a $1,700 ACTC."
+us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model approximated the 2026 one-child EITC maximum as $4,425 instead of $4,427, which left the total $2 short."
+us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a second qualifying child, but only the 10-year-old qualifies for EITC or CTC. That led it to apply a two-child EITC of $7,830 and two $1,700 ACTCs, instead of the one-child $4,427 EITC plus one $1,700 ACTC."
+us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave a one-child EITC of $6,936, above the 2026 one-child maximum of $4,427. It also set the ACTC to $0 on the claim that regular tax absorbs the CTC, but $22,000 of AGI is below the joint standard deduction, so tax is $0 and the full $1,700 is refundable."
us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts did not support eligibility and never ran either qualifying test. At $22,000 for a family of three, income is 81% of the poverty guideline, under the 130% free-meal limit. The household's SNAP receipt of about $407.90 a month separately makes the child categorically eligible, so the correct tier is FREE (1), not 0."
us,scenario_012,free_school_meals_eligible,grok-4.7,llm_error,age_disability,False,"The model applied the rule that unlisted booleans are false to K-12 enrollment and set it to false. PolicyEngine does not take enrollment as an input here; it derives it from age, so a 10-year-old counts as a K-12 student. With enrollment in place, SNAP categorical eligibility and income at 81% of the poverty guideline both put the child in the FREE tier."
us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model claimed income was above 130% of the poverty guideline, but $22,000 for a three-person household is only 81% of the guideline, well below the roughly $35,000 limit for that size. It also missed that SNAP receipt of about $407.90 a month makes the child categorically eligible for free meals whatever the income."
@@ -923,51 +935,52 @@ us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output
us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without first assigning the household to the free-meals tier. The household's 0.81 FPG ratio and categorical eligibility produce FREE status, which makes reduced-price eligibility false."
us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that income below 130% FPG places the child in the free-meal range and that reduced-price support applies only from 130% through 185%, but then submitted 1 contrary to its own derivation. FREE-tier classification, independently reinforced by categorical eligibility, makes the reduced-price output 0."
-us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model gave the energy subsidy a ~$400 utility allowance, but LIHEAP receipt does not trigger one for a household with no elderly or disabled member. It also used a $1,023 maximum allotment. Its final $378/month is exactly the household's 30% contribution, not the $785 − $378 = $407 allotment."
-us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The deductions ($209 standard, $367 earned income) and the no-shelter conclusion are right. However, the model left the benefit at $407.70 instead of applying SNAP rounding: net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407/month, or $4,884."
-us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left out the 20% earned income deduction entirely. It also used an outdated $256 standard deduction and a $694 maximum allotment. It then treated the excluded $411 energy subsidy as reducing net income, which cut the benefit to $84/month instead of $407."
-us,scenario_012,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached ~$412/month using a $224 standard deduction instead of $209, then dropped that result for an unsupported $601/month. The correct figures are $785 minus 30% of $1,258 (rounded up to $378), which gives $407/month."
-us,scenario_012,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly found no shelter deduction and got about $406/month, using a $204 standard deduction instead of $209. It then raised the result to $514/month by 're-estimating with shelter', even though it had already shown rent was below half of net income."
-us,scenario_012,snap,claude-opus-5,llm_error,other,False,"The model reported $378/month as the SNAP benefit. That is the household's expected contribution (30% of $1,258 net income, rounded up), not the allotment, which is $785 − $378 = $407/month."
-us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as qualifying for a ~$395 utility allowance and took a $216 excess shelter deduction. Under the 2026 rule, LIHEAP receipt gives a utility allowance only to households with an elderly or disabled member, so only the $450 rent counts, and it is below half of the $1,258 net income."
-us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a fourth SNAP member (SNAP counts only born members) and used the four-person $973 maximum. It also added a LIHEAP-triggered utility allowance the household does not qualify for. Its submitted $5,136 does not even match its own $7,968 derivation."
-us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model treated the pregnant spouse's unborn child as a SNAP member, making a household of four with a ~$994 maximum. It also added a utility allowance from the energy subsidy. The correct unit is three people with a $785 maximum and no shelter deduction."
-us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"The model's structure is correct (net $1,258, no shelter deduction), but it rounded the 30% contribution down to $377 instead of up to $378. That gave $408/month instead of $407, and $4,890 is not even 12 × $408."
-us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $211 standard deduction and a $786 maximum allotment instead of the FY2026 three-person figures of $209 and $785. It also skipped rounding the 30% contribution up, giving $409/month instead of $407."
-us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added a $500 utility allowance to rent and took a $316.67 excess shelter deduction. This household has no elderly or disabled member, so its LIHEAP subsidy does not trigger a utility allowance and there is no shelter deduction. The model also used a $200 standard deduction instead of $209."
-us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,The model used the FY2025 standard deduction ($204) and maximum allotment ($768) instead of the FY2026 figures ($209 and $785). It also left the benefit unrounded at $389.20 instead of computing $785 − $378 = $407/month.
-us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child and used a four-person SNAP household and maximum allotment. SNAP counts only the three born members, which gives a $785 maximum and $407/month."
-us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy triggers a Standard Utility Allowance and took an excess shelter deduction, giving $482/month. For a household with no elderly or disabled member, LIHEAP receipt does not confer a utility allowance, so the $450 rent alone produces no shelter deduction."
-us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $446/month implies net income of about $1,130, roughly $128 below the correct $1,258. That matches an excess shelter deduction the household is not entitled to, since the $450 rent is below half of net income and no utility allowance applies. The correct figure is $785 − $378 = $407/month."
-us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model included a Standard Utility Allowance in shelter costs and took an excess shelter deduction. Under the 2026 rule, a household with no elderly or disabled member gets no utility allowance from LIHEAP receipt, so there is no shelter deduction and the benefit is $407/month."
-us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction, giving $457/month. Shelter costs are only the $450 rent, which is below half of the $1,258 net income, because the energy subsidy does not trigger a utility allowance for this household."
-us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"The model's $1,450/year (about $121/month) implies countable net income of about $2,213, which is above the household's $1,833 gross income. This is not a coherent use of the SNAP formula. The correct figure is $785 − 30% of $1,258 net income = $407/month."
-us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model took an excess shelter deduction, giving $453/month. The household's only shelter cost is $450 rent, which is below half of net income, and the LIHEAP subsidy does not give a utility allowance to a household with no elderly or disabled member."
-us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model counted shelter costs as a deduction, giving about $472/month. Rent of $450 is below half of the $1,258 net income and no utility allowance applies, so there is no shelter deduction and the benefit is $407/month."
-us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $200/month standard deduction and the FY2025 $768 maximum allotment ($9,216/year) instead of the FY2026 figures of $209 and $785. It also computed on annual totals without the monthly whole-dollar rounding."
-us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added a SNAP member for the pregnancy and used the four-person $975 maximum and a $238 standard deduction. It also added a LIHEAP-triggered utility allowance. The correct unit is three people with a $785 maximum, a $209 standard deduction, and no shelter deduction."
-us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model gave the $411 energy subsidy a utility allowance and took an excess shelter deduction, lowering net income to about $1,044. Without a utility allowance (no elderly or disabled member), net income stays at $1,258 and the benefit is $407/month."
-us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model credited 'housing/utility costs' as deductions, giving about $497/month. The household has no utility allowance, and its $450 rent is below half of net income, so there is no shelter deduction and the benefit is $407/month."
-us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0, saying SNAP eligibility flags were missing. Wages, household composition, and assets are enough to compute SNAP, and the household passes the gross and net income tests with a $407/month allotment."
-us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy qualifies the household for the heating/cooling utility allowance and applied an excess shelter deduction. Under the 2026 rule, that pathway applies only to households with an elderly or disabled member, so net income is $1,258 and the benefit is $407/month, not $483."
-us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"The model correctly applied no utility allowance and no shelter deduction, but rounded $785 − $377.40 to $408. SNAP rounds 30% of net income up to $378, which gives $407/month, or $4,884."
-us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included a heating/cooling utility deduction from the energy subsidy and a rent-based shelter deduction, giving $478/month. The household has no elderly or disabled member, so it gets no utility allowance, and rent alone is below half of net income."
-us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"The model got $408/month by rounding $785 − $377.40 to the nearest dollar. SNAP rounds the 30% contribution up to $378, which gives $407/month."
-us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"The model correctly ruled out the utility allowance and shelter deduction, but computed on annual totals ($9,420 − 30% × $15,092). SNAP is computed monthly: net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407, giving $4,884 for the year."
-us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $406.20/month implies net income of $1,262.67, which means it used the FY2025 $204 standard deduction instead of $209. It also skipped rounding the 30% contribution up. The correct result is $407/month."
-us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model applied a shelter deduction, giving $504/month. Rent of $450 is below half of the $1,258 net income and no utility allowance applies, so there is no shelter deduction."
-us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model added an estimated $433 heating utility allowance 'triggered by the energy subsidy'. For a household with no elderly or disabled member, LIHEAP receipt no longer confers a utility allowance, so there is no excess shelter deduction and the benefit is $407/month."
-us,scenario_012,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $300/month matches $785 − 30% × ($1,833 − $209), which leaves out the 20% earned income deduction of $366.67. With that deduction, net income is $1,258 and the benefit is $407/month."
-us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used outdated parameters: a $198 standard deduction and a $766 maximum allotment (FY2024-era figures) instead of FY2026's $209 and $785. It also annualized $385 as $4,625 instead of $4,620."
-us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model included a 'LIHEAP-triggered SUA' in an excess shelter deduction, giving $456/month. That trigger applies only to households with an elderly or disabled member, so this household has no shelter deduction and receives $407/month."
-us,scenario_012,snap,grok-4.7,llm_error,other,False,"The model used the correct $209 and $785 parameters but computed on annual totals without SNAP's monthly rounding. Net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407 × 12 = $4,884."
-us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted the $210 monthly standard deduction once from annual income instead of 12 times, which overstated net income by about $2,300 a year. It also added the energy subsidy to shelter costs and used a $9,422 annual maximum instead of 12 × $785."
-us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so nothing was scored against the $4,884 reference."
-us,scenario_012,snap,kimi-k3,llm_error,other,False,"The deductions and the no-shelter conclusion are correct, but the model left the benefit at $407.70. SNAP rounds net income to $1,258 and 30% of it up to $378, which gives $407/month, or $4,884."
-us,scenario_012,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model stopped before doing any calculation and output $0. The household passes the gross and net income tests, with net income of $1,258 and an allotment of $785 − $378 = $407/month."
-us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model added a 'LIHEAP-funded standard utility allowance' of about $470 and a ~$290 excess shelter deduction. With no elderly or disabled member, LIHEAP receipt does not confer a utility allowance, so net income is $1,258, not ~$985."
-us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted the $411 energy subsidy as unearned income, but energy assistance is excluded from SNAP income. It also used an outdated $768 maximum and a $198 standard deduction, then changed its own $4,525 result to $426/month without justification."
-us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model set net income equal to gross earnings and left out the $209 standard deduction and the 20% earned income deduction. It then compared that against an outdated $21,080 poverty line. Correct net income is $1,258/month, well under the $2,220.83 net income limit."
+us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the household's expected contribution ($378, which is 30% of net income) as the monthly benefit, instead of subtracting it from the $785 maximum to get $407. Its working also wrongly added a utility allowance (about $400) because of the energy subsidy, and used a $1,023 maximum allotment."
+us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The deductions, the $785 maximum and the 'no shelter deduction' finding were all right, but the model skipped SNAP's rounding rules. Rounding 30% of the $1,258 net income up gives a $378 contribution and a whole-dollar benefit of $407. Using the unrounded $407.70 inflated the annual total to $4,892."
+us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left out the 20% earned-income deduction ($366.67) and used wrong amounts: a $256 standard deduction instead of $209 and a $694 maximum allotment instead of $785. It then treated the $411 energy subsidy as something that raises income, which cut the monthly benefit to $84 instead of $407."
+us,scenario_012,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model wrongly found the household over the income limits, partly by counting four people. Gross income of $1,833 is far below the 130% limit of about $2,887 for three people, and net income of $1,258 is below the $2,221 poverty line, so the household qualifies for $407 a month."
+us,scenario_012,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Using a $224 standard deduction instead of $209, the model got about $412 a month with no shelter deduction. It then replaced that with an unexplained $601 a month. The correct net income is $1,258 with no shelter deduction, giving $785 − $378 = $407."
+us,scenario_012,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model first found no shelter deduction and about $406 a month (using a $204 standard deduction instead of $209). It then 're-estimated with shelter' to $514 a month. No excess shelter deduction applies because the $450 rent is less than half of the $1,258 net income and the energy subsidy gives no utility allowance."
+us,scenario_012,snap,claude-opus-5,llm_error,other,False,"The model reported $378 a month, which is the 30% contribution from the $1,258 net income, as the benefit itself. It never subtracted that from the $785 maximum, which leaves $407."
+us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed the energy subsidy entitles the household to a standard utility allowance (about $395) and took a $216 excess shelter deduction. Under the 2026 rules, energy assistance gives that allowance only to households with an elderly or disabled member. Shelter cost is therefore just the $450 rent, below half of the $1,258 net income, so there is no deduction."
+us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a fourth SNAP member, using four-person amounts ($973 maximum) and FY2025 figures. It also added an energy-subsidy utility allowance of about $414. It then submitted $5,136, which does not match its own $7,968 calculation. The correct unit is three people with no shelter deduction, which gives $407 a month."
+us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model treated the pregnant spouse as adding a fourth member (about $994 maximum allotment) and added a utility allowance because of the energy subsidy. SNAP does not count an unborn child, and this household gets no utility allowance, so the correct figure is the three-person $785 maximum minus $378."
+us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"The model correctly found net income of $1,258 and no shelter deduction, but rounded the result to about $408 a month. SNAP rounds 30% of net income up to $378, which gives $785 − $378 = $407, or $4,884 a year."
+us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $211 standard deduction and a $786 maximum allotment. The FY2026 three-person amounts are $209 and $785, which give net income of $1,258, a $378 contribution and a $407 benefit, not $409."
+us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added a $500 standard utility allowance to the rent, producing a $316.67 excess shelter deduction, and used a $200 standard deduction. The household has no utility allowance, so shelter cost is only $450, below half of the $1,258 net income, and the benefit is $407 a month."
+us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used outdated FY2025 amounts: a $204 standard deduction and a $768 maximum allotment. With the FY2026 amounts of $209 and $785, net income is $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a member and used a four-person household. SNAP counts only the three people present, so the $785 three-person maximum applies and the benefit is $407 a month."
+us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy triggered a standard utility allowance and applied an excess shelter deduction. For a household with no elderly or disabled member, energy assistance does not give a utility allowance in 2026. The $450 rent alone is below half of the $1,258 net income, so no shelter deduction applies."
+us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave $446 a month with no derivation. That is $39 above the correct $407, which matches an unsupported deduction cutting net income by about $130. No shelter deduction applies, so net income stays at $1,258 and the contribution at $378."
+us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction that included a standard utility allowance. The household pays no utilities, and the energy subsidy does not give a utility allowance to a non-elderly, non-disabled household. Shelter cost is therefore just $450 and no excess shelter deduction applies."
+us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied a shelter deduction that lowered net income enough to produce $457 a month. Only the $450 rent counts as a shelter cost, which is below half of the $1,258 net income, so the benefit is $785 − $378 = $407."
+us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"The model gave $1,450 a year with no derivation, which implies a contribution of about $664 a month. That is consistent with using roughly gross income instead of net income after the $209 standard deduction and the 20% earned-income deduction. The correct net income is $1,258, giving a $378 contribution and a $407 monthly benefit."
+us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction that does not exist here. With only the $450 rent counted, half of the $1,258 net income ($629) is higher, so the benefit stays at $407 a month."
+us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model included shelter costs as a deduction, implying a utility allowance from the energy subsidy. That allowance is not available to this household, and the $450 rent alone produces no excess shelter deduction, so the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $200 monthly standard deduction ($2,400 a year) and the FY2025 $768 maximum ($9,216 a year). The FY2026 amounts are $209 ($2,508 a year) and $785 ($9,420 a year), which give a $407 monthly benefit."
+us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added a fourth member for the pregnancy, which led it to use a $975 maximum and a $238 standard deduction. It also applied a $360 utility allowance because of the energy subsidy. SNAP counts three people here and gives no utility allowance, so the benefit is $785 − $378."
+us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model added a utility allowance based on the energy subsidy to the $450 rent, creating an excess shelter deduction and $1,044 net income. It also used a $204 standard deduction instead of $209. Without the utility allowance there is no shelter deduction, so net income is $1,258 and the benefit is $407."
+us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model counted housing and utility costs toward a shelter deduction. The household has no utility allowance, and the $450 rent is below half of the $1,258 net income, so no deduction applies and the benefit is $407 a month."
+us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned 0, saying more eligibility inputs were needed. The facts given are enough: a three-person household with $1,833 monthly earnings and $2 in assets passes the gross and net income tests and receives $407 a month."
+us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed the energy subsidy qualifies the household for the heating and cooling utility allowance. Under the 2026 rules, energy assistance gives that allowance only to households with an elderly or disabled member, so no excess shelter deduction applies and net income stays at $1,258."
+us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"The model correctly used no utility allowance, but got $408 a month. SNAP rounds 30% of the $1,258 net income up to $378, which leaves a $407 benefit, or $4,884 a year."
+us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied a heating and cooling utility deduction on top of the rent. This household does not qualify for that allowance, so shelter cost is only $450, below half of net income, and the benefit is $407 a month, not $478."
+us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"The model reported $408 a month instead of $407. SNAP rounds net income to $1,258 and the 30% contribution up to $378, giving $785 − $378 = $407 and $4,884 a year."
+us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"The model correctly ruled out the energy-subsidy utility allowance and the shelter deduction, but worked on annual totals without SNAP's monthly rounding. Monthly net income rounds to $1,258, its 30% contribution rounds up to $378, and the benefit is $407 a month, or $4,884, not $4,892.40."
+us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $406.20 monthly benefit implies a contribution of $378.80 and net income of $1,262.67, which means it used the FY2025 $204 standard deduction instead of the FY2026 $209. It also skipped rounding. The correct figures are net income $1,258, contribution $378 and benefit $407."
+us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model included a shelter deduction. With no utility allowance, the $450 rent is below half of the $1,258 net income, so no shelter deduction applies and the benefit is $407 a month, not $504."
+us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct $785 maximum and $209 standard deduction, but added a $433 heating utility allowance because of the energy subsidy. Energy assistance does not give a utility allowance to this non-elderly, non-disabled household in 2026, so no shelter deduction applies and the benefit is $407 a month."
+us,scenario_012,snap,grok-4.3,llm_error,other,False,"The model gave $3,600 ($300 a month) with no calculation. The correct result is $785 minus a $378 contribution on $1,258 net income, which is $407 a month. Its figure understates the benefit by about $107 a month."
+us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used a $198 standard deduction and a $766 maximum allotment instead of the FY2026 $209 and $785, getting $385 a month. It also multiplied $385 by 12 wrongly, reporting $4,625 instead of $4,620."
+us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model gave the household a standard utility allowance triggered by the energy subsidy and an excess shelter deduction. In 2026 that allowance requires an elderly or disabled member, so shelter cost is only $450, below half of net income, and the benefit is $407 a month."
+us,scenario_012,snap,grok-4.7,llm_error,other,False,"The deductions, the $785 maximum and the lack of a shelter deduction were all right, but the model calculated on annual totals without SNAP rounding. Net income rounds to $1,258 a month and the 30% contribution rounds up to $378, giving $407 a month and $4,884, not $4,892.40."
+us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted the $210 monthly standard deduction from annual income instead of the annual $2,508 ($209 × 12). This overstated net income by about $2,300 a year and inflated the contribution. It also counted the $411 energy subsidy as a shelter cost."
+us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so there was no answer to score."
+us,scenario_012,snap,kimi-k3,llm_error,other,False,"The derivation (net income $1,257.67, no shelter deduction, $785 maximum) is right, but the model skipped SNAP's rounding. Net income rounds to $1,258, the 30% contribution rounds up to $378, and the benefit is $407 a month, not $407.70."
+us,scenario_012,snap,minimax-m3,llm_error,other,False,"The model returned 0 without doing any calculation. Gross income of $1,833 passes the three-person gross limit and net income of $1,258 passes the net test, so the benefit is $785 − $378 = $407 a month."
+us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model applied a standard utility allowance of about $470 funded by energy assistance (LIHEAP), creating a $290 excess shelter deduction, and used a $204 standard deduction. Energy assistance gives no utility allowance to this non-elderly, non-disabled household, so net income is $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted the $411 energy subsidy as income and used a $198 standard deduction and a $768 maximum allotment. It then switched to an unexplained $426 a month. Energy assistance is excluded from SNAP income, and the FY2026 amounts are $209 and $785, which give $407 a month."
+us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used the full $22,000 gross earnings as net income for the net income test. It skipped the 20% earned-income deduction and the $209 standard deduction, which bring net income down to $1,258 a month, well below the $2,221 poverty line."
us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It omitted the prior Medicaid screen: the 18-year-old spouse qualifies for Medicaid under the OLDER_CHILD category, which makes her ineligible for CHIP."
us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age once income requirements are met. It failed to apply the rule excluding anyone already Medicaid-eligible; this spouse is Medicaid-eligible as an OLDER_CHILD.
us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified low-income pregnancy coverage as CHIP-funded coverage without first testing Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP, irrespective of household size adjustments for the unborn child or pregnancy income limits."
@@ -991,123 +1004,126 @@ us,scenario_013,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,t
us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model asserted that income was ""near SSI/FPL-based aged-disabled standards"" but never compared it to a limit. The head's countable income is about $30,5xx: $23,736 of Social Security plus the $6,720 pension and $88 of dividends and capital gains, less the $20 general disregard. That is about 1.9x FPL, well above the 100% FPL aged/disabled income standard and the SSI benefit rate. SSI is 0, so no SSI-linked pathway applies, and at age 80 the head is outside the MAGI adult group."
us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model said Social Security is ""mostly excluded via SSI methodology"" and cut it to $10,196 countable. SSI methodology counts Social Security in full as unearned income, minus only the $20 general disregard, so countable income is about $30,5xx, or 1.9x FPL, above the aged/disabled standard. The model also applied the ALTCS 300%-of-SSI limit, which requires an institutional level-of-care need this household does not have. It then deducted health premiums and OTC costs as a spend-down, which does not create eligibility here, and it ignored its own finding that the $58,700 in assets exceeds the $2,000 limit."
us,scenario_013,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model tested AGI of $6,808 against 138% FPL. MAGI adds back the $23,736 of nontaxable Social Security, which puts income at about $30,544, or 1.91x FPL. The 138% FPL adult expansion group also covers only ages 19-64, so it cannot apply to an 80-year-old. The aged/disabled pathway counts total income including Social Security, and that income is far above its limit."
-us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied only the regular elderly/disabled tests: the 100% FPL net income limit and the ~$4,500 asset limit. It never applied Arizona's March 2026 expansion of categorical eligibility to 200% FPL ($2,608.33/mo), which covers its ~$2,545 of income and waives both tests. As a categorically eligible one-person household, it gets the $24 minimum for March–December, or $240."
-us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Denied eligibility because net income (~$2,322) was above the 100% FPL net limit. It missed that Arizona's categorical eligibility at 200% FPL, in effect from March 2026, covers the $2,539 of gross income and treats the net income test as met. The categorically eligible single-person household receives the $24 minimum allotment for 10 months, or $240."
-us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test and a $2,500 asset limit. Elderly/disabled households are exempt from the gross test, and Arizona's categorical eligibility at 200% FPL from March 2026 ($2,608.33/mo) waives the asset test for this $2,539/mo household. It therefore missed the $24/month minimum for March–December ($240)."
-us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Correctly skipped the gross test for an elderly/disabled household, but then treated the 100% FPL net income limit as binding. Arizona's categorical eligibility at 200% FPL from March 2026 treats that test as met. The benefit formula comes out below zero, so the categorically eligible one-person household gets the $24 minimum for 10 months, or $240."
-us,scenario_013,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Denied SNAP solely because the $58,700 in the bank exceeds the ~$4,500 elderly/disabled resource limit. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) removes the asset test for this $2,539/mo household. That leaves $24/month minimum benefits for March–December ($240)."
-us,scenario_013,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Treated the regular SNAP asset limit as disqualifying. Under Arizona's categorical eligibility at 200% FPL ($2,608.33/mo from March 2026), the $2,539/mo household is categorically eligible and the asset test does not apply. The one-person household is therefore owed the $24 minimum allotment for 10 months ($240)."
-us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Ended at the 100% FPL net income test (net ~$2,330 vs ~$1,304), without recognizing that Arizona's categorical eligibility at 200% FPL from March 2026 treats the net income test as met. It also missed that categorically eligible one-person households receive the $24 minimum even when the formula is negative, which gives $240 for March–December."
-us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Denied eligibility using the regular net income test and the $4,500 asset limit. It also misread the $298 annual premium as a monthly amount. It never applied Arizona's categorical eligibility at 200% FPL (from March 2026), which covers $2,539/mo of income and waives both tests, leaving the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Invoked a 200% FPL categorical eligibility limit but set it at ~$2,510/mo instead of $2,608.33 (200% of the FY2026 $1,304.17 guideline). It also counted the $72 of capital gains, putting gross income at $2,545 instead of $2,539.33. With the correct figures the household passes, the asset test is waived, and it receives the $24 minimum for March–December ($240)."
-us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Correctly found that $298 minus 30% of ~$2,330 net income is below zero, but stopped there. A one-person household made categorically eligible by Arizona's 200% FPL expansion (from March 2026) receives the $24 minimum allotment regardless, which totals $240 over March–December."
-us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Applied the regular elderly/disabled asset limit to the $58,700 bank balance. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) covers this $2,539/mo household and waives the resource test. The result is the $24/month minimum for 10 months ($240)."
-us,scenario_013,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Used the $4,250 elderly/disabled asset limit as disqualifying. From March 2026, Arizona's categorical eligibility at 200% FPL ($2,608.33/mo) covers $2,539.33 of gross income and eliminates the asset test. The household therefore receives the $24 one-person minimum for March–December ($240)."
-us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied both the regular net income limit and the elderly/disabled resource limit. Arizona's categorical eligibility at 200% FPL from March 2026 treats both as met for this $2,539/mo household. A categorically eligible one-person household receives the $24 minimum, or $240 over 10 months."
-us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Denied SNAP on the regular net income limit and asset limit, missing Arizona's categorical eligibility at 200% FPL effective March 2026, which covers $2,539/mo of gross income and waives both tests. The benefit formula is negative, so the one-person household gets the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 185% FPL categorical eligibility limit (~$2,321/mo, itself miscomputed) for Arizona. From March 2026 the limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income (capital gains excluded) passes. Categorical eligibility then yields the $24 minimum for March–December ($240)."
-us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Applied the regular one-person resource limit and net income test, ignoring Arizona's categorical eligibility at 200% FPL from March 2026, which covers the $2,539/mo household and waives both. The categorically eligible single-person household receives the $24 minimum allotment for 10 months ($240)."
-us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Treated the elderly net income limit as disqualifying. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met for gross income of $2,539/mo. The one-person household therefore gets the $24 minimum benefit for March–December ($240)."
-us,scenario_013,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Measured income against a 185% FPL categorical eligibility threshold and then applied the $4,250 asset limit. Arizona's limit is 200% FPL from March 2026 ($2,608.33/mo), and the $2,539.33 of countable income passes it, so the asset test is waived. The result is the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Applied the regular income and asset thresholds without recognizing Arizona's categorical eligibility at 200% FPL from March 2026, which covers $2,539/mo of gross income and treats the income and asset tests as met. The one-person household receives the $24 minimum for March–December ($240)."
-us,scenario_013,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Claimed gross income exceeds Arizona's threshold, but $2,539.33/mo is below Arizona's 200% FPL categorical eligibility limit ($2,608.33/mo from March 2026, $2,660 from October). Categorical eligibility then gives the $24 one-person minimum for 10 months ($240)."
-us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Relied on the net income limit and a zero formula benefit. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 treats the net income test as met, and that categorically eligible one-person households receive the $24 minimum even when the formula is negative, which gives $240 for March–December."
-us,scenario_013,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used a 185% FPL categorical eligibility limit for Arizona and then applied the federal elderly asset limit. From March 2026 Arizona's limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income passes, so the asset test is waived. The household receives the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit for elderly/disabled households as the sole test. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met for this $2,539/mo household. The one-person household receives the $24 minimum for March–December ($240)."
-us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"Applied gross and net income limits to a senior household, although elderly households face no gross test. It also counted $72 of capital gains as income. It missed Arizona's categorical eligibility at 200% FPL from March 2026, which covers $2,539.33/mo and deems the tests met, yielding the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Declared Social Security plus pension income too high under regular SNAP rules. Arizona's categorical eligibility at 200% FPL ($2,608.33/mo from March 2026) covers this $2,539/mo household. The categorically eligible one-person household receives the $24 minimum for March–December ($240)."
-us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Gave no substantive computation and defaulted to zero. The correct derivation compares $2,539.33/mo of gross income with Arizona's 200% FPL categorical eligibility limit ($2,608.33 from March 2026), which confers eligibility, and pays the $24 one-person minimum for 10 months ($240)."
-us,scenario_013,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Denied SNAP because $58,700 exceeds the elderly/disabled resource limit. Arizona's categorical eligibility at 200% FPL from March 2026 waives the asset test for this $2,539/mo household, leaving the $24 minimum allotment for March–December ($240)."
-us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Concluded the allotment formula yields zero but omitted the minimum benefit. Under Arizona's categorical eligibility at 200% FPL from March 2026, the one-person household is categorically eligible and receives the $24 minimum each month, or $240 for March–December."
-us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Stopped at the benefit formula producing no positive allotment. A one-person household that is categorically eligible through Arizona's 200% FPL expansion (from March 2026) receives the $24 minimum allotment regardless, totaling $240."
-us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the regular net income limit, missing that Arizona's categorical eligibility at 200% FPL from March 2026 deems the net income test met for $2,539/mo of gross income. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Asserted that income exceeds Arizona's categorical eligibility limit, which means it used the older 185% FPL cap. From March 2026 the limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income passes, so the asset test is waived and the $24 minimum is paid for March–December ($240)."
-us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Concluded no positive allotment is produced after deductions and omitted the $24 minimum benefit. That minimum applies to one-person households made categorically eligible by Arizona's 200% FPL expansion from March 2026, totaling $240."
-us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Treated a negative formula benefit as zero SNAP. A one-person household categorically eligible under Arizona's 200% FPL limit (from March 2026) receives the $24 minimum allotment each month, or $240 for March–December."
-us,scenario_013,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"Claimed income exceeds Arizona's categorical eligibility limit, applying the pre-2026 185% FPL cap. From March 2026 the cap is 200% FPL ($2,608.33/mo), which $2,539.33 of countable income passes. That waives the asset test and yields the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"Applied the regular income and asset limits. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 covers this $2,539/mo household and treats both tests as met. The household gets the $24 one-person minimum for March–December ($240)."
-us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"Denied SNAP on the 100% FPL net income test, which Arizona's categorical eligibility at 200% FPL (from March 2026) deems met for $2,539/mo of gross income. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"Applied the regular net income limit and the elderly/disabled resource limit. It missed Arizona's categorical eligibility at 200% FPL from March 2026, which waives both for this $2,539/mo household. That produces the $24 minimum for March–December ($240)."
-us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the $298 maximum, but missed that one-person households categorically eligible under Arizona's 200% FPL expansion (from March 2026) receive the $24 minimum allotment anyway. That totals $240 for 10 months."
-us,scenario_013,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Treated the $4,250 elderly/disabled resource limit as disqualifying. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) covers the $2,539/mo household and removes the asset test, leaving the $24 minimum for March–December ($240)."
-us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"Denied SNAP because net income (~$2,310) exceeds the 100% FPL limit, without applying Arizona's categorical eligibility at 200% FPL from March 2026, which deems that test met. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Applied the federal elderly/disabled resource limit and the 100% FPL net income test. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 covers $2,539/mo of gross income and treats both tests as met, producing the $24 minimum for March–December ($240)."
-us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"Ended at the one-person net income limit, missing that Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met. It also omitted the $24 minimum allotment that categorically eligible one-person households receive, which totals $240 over 10 months."
-us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"Applied the regular income and asset thresholds for an elderly household. Arizona's categorical eligibility at 200% FPL from March 2026 covers this $2,539/mo household and waives those tests, yielding the $24 minimum for March–December ($240)."
-us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly exempted the household from the gross test but denied it on the 100% FPL net income test. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met, and the one-person household receives the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test, which does not apply to elderly/disabled households. It also missed Arizona's categorical eligibility at 200% FPL from March 2026 ($2,608.33/mo), which the $2,539/mo household passes, leaving the $24 minimum for March–December ($240)."
-us,scenario_013,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Invented a rule that households made up entirely of people aged 60+ are ineligible for SNAP; no such exclusion exists, and elderly households get more favorable treatment. Through Arizona's categorical eligibility at 200% FPL from March 2026, this household is eligible for the $24 one-person minimum for 10 months ($240)."
+us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied only the regular elderly/disabled rules: the 100% FPL net income test and the asset limit. It never checked Arizona's BBCE. From March 2026, gross income of about $2,539/month is under the 200% FPL limit ($2,608), so both tests count as met. As a one-person household it then gets the $24 minimum for 10 months."
+us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Concluded the household is ineligible because net income (~$2,322) exceeds the 100% FPL net limit. It missed that Arizona's BBCE covers gross income up to 200% FPL ($2,608/month) from March 2026 and waives the net income test. A categorically eligible one-person household gets the $24 minimum benefit, which is $240 for March–December."
+us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test and a $2,500 asset limit. An elderly/disabled household is exempt from the gross test. More importantly, Arizona's BBCE at 200% FPL from March 2026 makes this household categorically eligible and waives the asset test, so it gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Denied benefits because the aged/disabled income and asset tests were not met. It never considered Arizona's BBCE: from March 2026, gross income of $2,539/month is under 200% FPL ($2,608), which waives those tests. The result is the $24 one-person minimum benefit for 10 months, or $240."
+us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Ran only the regular net income test (net ~$2,265 vs ~$1,255) and stopped at ineligible. It missed Arizona's March 2026 BBCE limit of 200% FPL ($2,608/month gross), which this household meets. That waives the net income test and entitles a one-person household to the $24 minimum benefit for March–December."
+us,scenario_013,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Treated the $58,700 bank balance as disqualifying under the ~$4,500 elderly/disabled resource limit. Arizona's BBCE removes the asset test for households with gross income up to 200% FPL from March 2026, and this household's $2,539/month qualifies. It then receives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Denied SNAP over the asset limit and never applied Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL ($2,608/month), so the asset and net income tests count as met. The household then gets the $24 one-person minimum benefit, which is $240 for the year."
+us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Concluded zero because net income (~$2,330) exceeds 100% FPL. It missed that Arizona's BBCE (200% FPL gross limit from March 2026) waives the net income test. A categorically eligible one-person household whose formula benefit is below zero still receives the $24 minimum, giving $240 for 10 months."
+us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Applied the regular net income test and the $4,500 elderly/disabled asset limit, and treated the $298 annual premium as a monthly expense. It never applied Arizona's BBCE: gross income of $2,539/month is under the 200% FPL limit from March 2026, which waives both tests. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Correctly tested against a 200% FPL gross limit, but computed it as ~$2,510/month from the stale 2024 poverty guideline and added the $72 capital gains into gross income. The correct limit is $2,608.33 from March 2026 ($2,660 from October), and gross income of $2,539 falls under it. That makes the household categorically eligible for the $24 minimum for 10 months."
+us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Took $298 minus 30% of net income going negative to mean a zero benefit. It missed that the household is categorically eligible through Arizona's BBCE (gross under 200% FPL from March 2026). Eligible one-person households receive the $24 minimum even when the formula is below zero, so the answer is $240 for March–December."
+us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Denied SNAP over the elderly/disabled asset limit. From March 2026, Arizona's BBCE covers gross income up to 200% FPL ($2,608/month), which this household meets, and that waives the asset test. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Applied the $4,250 elderly/disabled asset limit as decisive. Arizona's BBCE at 200% FPL from March 2026 makes the household categorically eligible, which means the asset limit does not apply. The household then receives the $24 one-person minimum benefit for 10 months, or $240."
+us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Relied on the regular net income and resource tests. It missed that from March 2026, Arizona's BBCE treats both tests as met for gross income under 200% FPL ($2,608/month vs. the household's $2,539). That entitles the one-person household to the $24 minimum for 10 months."
+us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Concluded ineligibility from the net income limit and the asset limit. Both tests are waived under Arizona's BBCE, which covers gross income up to 200% FPL from March 2026. The household then receives the $24 one-person minimum benefit for March–December, or $240."
+us,scenario_013,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used Arizona's old 185% FPL gross limit (~$2,321/month). From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/month), and the household's $2,539 gross income meets it. That waives the net income test and yields the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Denied SNAP under the regular resource limit and net income test. It never applied Arizona's BBCE, which from March 2026 treats both tests as met for gross income up to 200% FPL. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Applied only the elderly net income limit. Arizona's BBCE (200% FPL gross from March 2026) makes this household categorically eligible and waives that limit. A categorically eligible one-person household receives the $24 minimum benefit for 10 months, or $240."
+us,scenario_013,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"Tested BBCE against Arizona's old 185% FPL limit, concluded the household fell outside it, and then applied the $4,250 asset limit. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608/month), and $2,539 gross income qualifies. That waives the asset test and yields the $24 minimum for 10 months."
+us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Denied SNAP on income and asset thresholds without applying Arizona's BBCE. From March 2026, gross income under 200% FPL ($2,608/month) makes the household categorically eligible. It then receives the $24 one-person minimum for March–December."
+us,scenario_013,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Asserted that gross income exceeds Arizona's eligibility threshold. From March 2026, Arizona's BBCE gross limit is 200% FPL ($2,608.33/month), above the household's $2,539.33. The household is categorically eligible for the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Treated a formula result at or below zero as a zero benefit. Under Arizona's BBCE (200% FPL from March 2026), the household is categorically eligible and the net income test is waived. Eligible one-person households receive the $24 minimum benefit, which comes to $240 for 10 months."
+us,scenario_013,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Applied Arizona's BBCE at the old 185% FPL limit and then the elderly asset limit. From March 2026, the BBCE limit is 200% FPL ($2,608/month), and $2,539 gross income meets it. That waives the asset test and entitles the household to the $24 one-person minimum for 10 months."
+us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"Compared gross income with the 100% FPL net income limit and stopped. It missed Arizona's BBCE: from March 2026, gross income under 200% FPL makes the household categorically eligible, and the household then gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"Computed net income of ~$27,012/year (including $72 of capital gains in income) and denied SNAP for exceeding the gross and net limits. An elderly/disabled household is exempt from the gross test, and Arizona's BBCE (200% FPL from March 2026) waives the net test. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Called the income too high without checking Arizona's BBCE. From March 2026, gross income of $2,539/month is under the 200% FPL limit ($2,608), so the household is categorically eligible and receives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to zero without any eligibility analysis. The correct path: monthly gross income of $2,539.33 falls under Arizona's 200% FPL BBCE limit from March 2026, which waives the income and asset tests. That yields the $24 one-person minimum for 10 months, or $240."
+us,scenario_013,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Denied SNAP because $58,700 exceeds the elderly/disabled resource limit. From March 2026, Arizona's BBCE covers gross income up to 200% FPL and waives the resource test. The household receives the $24 one-person minimum for March–December."
+us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Took the formula allotment (below zero) as the final answer. As a household categorically eligible through Arizona's 200% FPL BBCE from March 2026, a one-person household receives the $24 minimum benefit instead of zero, or $240 for 10 months."
+us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Concluded that income after deductions produces no allotment. It missed that BBCE eligibility (gross under Arizona's 200% FPL limit from March 2026) entitles a one-person household to the $24 minimum benefit even when the formula is negative, or $240 for 10 months."
+us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,Applied the regular 100% FPL net income test. Arizona's BBCE (200% FPL gross limit from March 2026) treats that test as met for this household. The one-person household then receives the $24 minimum for 10 months.
+us,scenario_013,snap,gpt-6-astra,llm_error,state_local_rule,False,"Asserted that income exceeds Arizona's categorical eligibility limit, which reflects the pre-2026 185% FPL threshold. From March 2026 the limit is 200% FPL ($2,608.33/month), and the household's $2,539.33 qualifies. That waives the asset test and yields the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Equated a non-positive formula allotment with zero SNAP. The household is categorically eligible under Arizona's 200% FPL BBCE from March 2026, so as a one-person household it receives the $24 minimum benefit for 10 months, or $240."
+us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Concluded that income after deductions produces no allotment. It never applied Arizona's March 2026 BBCE expansion to 200% FPL, which makes the household eligible. It also missed the $24 one-person minimum benefit that replaces a negative formula result, for $240 over 10 months."
+us,scenario_013,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Asserted that income exceeds Arizona's expanded categorical eligibility limit, using the old 185% FPL cap. From March 2026 the limit is 200% FPL ($2,608.33/month), above the household's $2,539.33. That waives the resource test and gives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"Denied SNAP on regular income and asset limits. Arizona's BBCE at 200% FPL from March 2026 waives both tests for this household's $2,539/month gross income. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit after standard and medical deductions. That test is treated as met under Arizona's BBCE, which covers gross income up to 200% FPL from March 2026. The one-person household then receives the $24 minimum for March–December."
+us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"Denied SNAP on the net income limit and the elderly/disabled resource limit. Both are waived under Arizona's BBCE: gross income of $2,539/month is under 200% FPL ($2,608) from March 2026. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the $298 maximum, but treated that as a zero benefit. Through Arizona's 200% FPL BBCE from March 2026, the household is categorically eligible. A one-person household then receives the $24 minimum benefit, or $240 for 10 months."
+us,scenario_013,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Applied the $4,250 elderly/disabled resource limit as disqualifying. Arizona's BBCE waives the resource test for gross income up to 200% FPL from March 2026, and this household qualifies. It receives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"Applied the regular 100% FPL net income test (net ~$2,310 vs ~$1,255) and denied benefits. Arizona's BBCE (200% FPL gross from March 2026) waives that test. The household receives the $24 one-person minimum benefit for 10 months, or $240."
+us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Denied SNAP on the federal elderly/disabled resource limit and the 100% FPL net income test. Arizona's BBCE covers gross income up to 200% FPL from March 2026, and that treats both tests as met. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"Applied the one-person net income limit to net income of ~$2,305 and denied benefits. Under Arizona's BBCE (gross under 200% FPL from March 2026), the net income test counts as met. The household receives the $24 one-person minimum for March–December."
+us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"Treated income and assets as exceeding elderly SNAP thresholds. It missed Arizona's March 2026 BBCE expansion to 200% FPL, which this $2,539/month gross income meets and which waives both tests. The household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly exempted the elderly/disabled household from the gross test, then denied benefits on the 100% FPL net income test. Arizona's BBCE at 200% FPL from March 2026 waives that net test. The household then receives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test, but elderly/disabled households are exempt from it. It also missed Arizona's BBCE at 200% FPL from March 2026, which makes this household categorically eligible for the $24 one-person minimum for 10 months."
+us,scenario_013,snap,qwen3.8-max,llm_error,age_disability,False,"Invented a rule that households of members aged 60+ are categorically ineligible for SNAP. In fact, elderly households get favorable treatment, and under Arizona's 200% FPL BBCE from March 2026 this household qualifies. It receives the $24 one-person minimum for 10 months."
us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Arizona’s 2.5% rate to $88 of dividends and capital gains without subtracting the Arizona standard deduction, which reduces taxable income to zero. It also violated its own computation by submitting $744.10 after its explanation calculated $2.20."
-us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Arizona has no refundable individual income tax credits at all. That ignores the refundable Credit for Increased Excise Taxes, which pays $25 per tax-unit member when a single filer's AGI is at or below $12,500. This filer's AGI is $6,736, so the credit is $25."
-us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Considered only the Family Income Tax Credit and the Property Tax Credit and concluded that no other refundable credit applies. It left out the refundable Increased Excise Tax Credit, which needs no dependents, rent or property tax: $25 for a one-person unit with AGI of $6,736, below the $12,500 single limit."
-us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Named the increased excise tax credit but lumped it with credits that require rent or property-tax payments, and so concluded that no conditions were met. The excise credit's only test is income (AGI at or below $12,500 single). The very low income the model itself identified ($6,736 AGI) qualifies the filer for $25."
-us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Claimed the increased excise tax credit yields nothing for a single filer with no dependents and no tax liability. The credit is refundable and pays $25 for the taxpayer alone whenever AGI is at or below $12,500. At $6,736 AGI the filer gets $25 despite having zero liability."
-us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Correctly identified the refundable excise tax credit and a qualifying AGI of about $6,800, but then counted $25 per exemption: personal, age 65+ and disability, giving $75. The credit is $25 per person in the tax unit (taxpayer, spouse, dependents), and age and disability exemptions do not add persons, so a one-person unit gets $25."
-us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Looked only for the family income tax credit and the property tax credit and concluded that nothing refundable applies. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member for single filers with AGI at or below $12,500, which this $6,736-AGI filer receives."
-us,scenario_013,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Checked only the property tax credit (no rent or property tax listed) and the family credit (no dependents). It skipped the refundable Increased Excise Tax Credit, which pays $25 to a single filer at $6,736 AGI with no expense or dependent requirement."
-us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Treated zero state tax liability and no dependents as ruling out refundable credits. The Increased Excise Tax Credit is refundable regardless of liability and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single limit."
-us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Assumed Arizona's refundable credits require dependents. The Increased Excise Tax Credit pays $25 per tax-unit member, including a lone taxpayer, when AGI is at or below $12,500, so this $6,736-AGI filer gets $25."
-us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Required dependents, donations or property-tax/rent facts for any refundable credit. It missed the income-tested Increased Excise Tax Credit, $25 for a one-person unit with AGI at or below $12,500, which applies at this filer's $6,736 AGI."
-us,scenario_013,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Framed Arizona's refundable credits around an EITC that requires earned income, but Arizona has no EITC. The applicable refundable credit is the Increased Excise Tax Credit, which needs no earnings and pays $25 to a single filer with AGI of $6,736."
-us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies without naming any credit test. It overlooked the refundable Increased Excise Tax Credit, $25 per tax-unit member when a single filer's AGI is at or below $12,500, which this $6,736-AGI filer meets."
-us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Stated that the household has no state refundable credits, which misses Arizona's refundable Increased Excise Tax Credit. That credit pays $25 for a one-person tax unit with AGI of $6,736, below the $12,500 single threshold."
-us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Asserted that no state refundable credit applies and never evaluated Arizona's Increased Excise Tax Credit. That credit is refundable and pays $25 per tax-unit member at or below $12,500 AGI for single filers, so this $6,736-AGI filer gets $25."
-us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies without evaluating the Increased Excise Tax Credit. That credit has only an income test, AGI at or below $12,500 single, which this filer's $6,736 AGI meets, yielding $25."
-us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Claimed Arizona has no major refundable credits this head qualifies for. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member for single filers with AGI at or below $12,500, which applies at $6,736 AGI."
-us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Asserted that household income exceeds the limits for Arizona's refundable credits. The Increased Excise Tax Credit tests AGI, which is $6,736 because Social Security is excluded, against a $12,500 single limit, so the filer qualifies for $25."
-us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Stated that Arizona has no refundable credits available to this household. It never applied the refundable Increased Excise Tax Credit, which pays $25 to a single filer with AGI of $6,736, below the $12,500 limit."
-us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies and overlooked the Increased Excise Tax Credit. That credit gives $25 per tax-unit member to single filers with AGI at or below $12,500, and this filer's AGI is $6,736."
-us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no triggering facts for an Arizona refundable credit. Low AGI alone ($6,736, below the $12,500 single limit) triggers the refundable Increased Excise Tax Credit of $25 per tax-unit member."
-us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed Arizona refundable credits hinge on qualifying dependents or earned income. The Increased Excise Tax Credit requires neither and pays $25 to a lone taxpayer with AGI at or below $12,500; here AGI is $6,736."
-us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Concluded that the facts support no Arizona refundable credit. The reported pension and investment income give AGI of $6,736, which qualifies the single filer for the refundable $25 Increased Excise Tax Credit (limit $12,500)."
-us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Required property tax, rent, dependents or other credit-generating facts. It missed that the refundable Increased Excise Tax Credit is purely income-tested and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single limit."
-us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Found no refundable credit eligibility for the one-person elderly household. The refundable Increased Excise Tax Credit pays $25 per tax-unit member when a single filer's AGI is at or below $12,500, and this filer's AGI is $6,736."
-us,scenario_013,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Concluded that the facts support no refundable Arizona credit and never evaluated the Increased Excise Tax Credit. That credit's only test is AGI at or below $12,500 for single filers, which $6,736 meets, yielding $25."
-us,scenario_013,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Required earnings, dependents or expenses to establish a refundable Arizona credit. The Increased Excise Tax Credit needs none of these and pays $25 to a single taxpayer with AGI of $6,736, below the $12,500 limit."
-us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Looked only at the Working Poor Tax Credit, which is a nonrefundable credit for charitable contributions. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member at AGI at or below $12,500 single, which this $6,736-AGI filer receives."
-us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Limited its analysis to the property-tax credit, which requires property taxes or rent. It skipped the refundable Increased Excise Tax Credit, which needs no expenses and pays $25 to a single filer with AGI of $6,736, below the $12,500 limit."
-us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded that household composition and income level rule out state refundable credits. A one-person unit with $6,736 AGI is exactly who the refundable Increased Excise Tax Credit targets: $25 at or below the $12,500 single limit."
-us,scenario_013,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated zero taxable income and no dependents as ruling out refundable credits. The Increased Excise Tax Credit is refundable even with no liability and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single threshold."
-us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Searched for a refundable Arizona EITC or property tax credit, but Arizona has no EITC and the filer lists no property tax. It missed the refundable Increased Excise Tax Credit, which pays $25 to a single filer with AGI of $6,736 and needs no earned income."
-us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Tested the excise tax credit against total household income of $30,544, which includes all $23,736 of Social Security, and a $25,000 single limit. The credit tests AGI, which excludes the non-taxable Social Security and is $6,736, against a $12,500 single limit, so the filer qualifies for $25."
-us,scenario_013,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Treated the absence of dependents as ruling out Arizona refundable credits. The Increased Excise Tax Credit pays $25 for the taxpayer alone when AGI is at or below $12,500, and this filer's AGI is $6,736."
-us,scenario_013,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Evaluated only the property tax refund credit, testing $30,544 of gross income (including non-taxable Social Security) against its low senior limit. It never considered the refundable Increased Excise Tax Credit, which tests AGI of $6,736 against $12,500 and pays $25."
-us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Required qualifying children or earned income and cited a nonexistent Arizona EITC. The refundable Increased Excise Tax Credit requires neither and pays $25 to a single filer with AGI of $6,736, below the $12,500 limit."
-us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Reasoned that with no Arizona tax liability there is nothing for a credit to offset. A refundable credit does not depend on liability, and the Increased Excise Tax Credit pays $25 to this single filer with $6,736 AGI, below the $12,500 limit."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the correct 2026 parameters ($32,200 deduction, 10% to $24,800), but it subtracted the $14,717 ESI premium from the stated gross wages as a pre-tax exclusion. That gave AGI of $74,211 instead of $88,927.65 and tax of $4,545. It then submitted $5,647, which matches neither its own figure nor the correct $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,000 MFJ standard deduction instead of the 2026 $32,200. It then taxed all $60,928 at a flat 12%, skipping the 10% bracket on the first $24,800. Finally it replaced that result with an unexplained $6,819. The correct $56,727.65 of taxable income yields $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the $32,200 deduction and $56,728 taxable income right, but it put the top of the 10% bracket at $24,150 instead of $24,800 and computed $6,324. It then dropped that figure for an unexplained $5,235 that no bracket schedule produces from $56,728 of taxable income."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own calculation (AGI $88,928, deduction about $32,300, 10%/12% brackets) came to $6,311, which matches the correct answer. It then submitted $7,466, a number that contradicts its own stated working."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It did the calculation exactly right: $32,200 deduction, $56,728 taxable income, and $2,480 + $3,831 = $6,311. It then submitted $6,712, citing unspecified 'estimated bracket thresholds', which overrides its own correct figure."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried forward pre-OBBBA 2025 parameters: a $30,000 MFJ standard deduction instead of $32,200, and a 10% bracket ending at $23,850 instead of $24,800. That overstated taxable income at $58,928 and produced $6,594 instead of $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It guessed a $32,600 deduction and a 10% bracket ending at $24,000, and computed about $6,279. It then submitted an unexplained $8,079, which is inconsistent with any 10%/12% calculation on roughly $56,000 of taxable income. The correct $32,200 deduction and $24,800 bracket yield $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 MFJ standard deduction instead of $32,200 and a 10% bracket ending at $24,400 instead of $24,800. That gave taxable income of $58,328 and tax of $6,511 instead of $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, so it used a $15,400 deduction, $10,000 of personal exemptions, and a 15% bracket. OBBBA made the TCJA structure permanent: the 2026 deduction is $32,200, there are no exemptions, and the second bracket is 12%. It also wrongly subtracted the $14,717 ESI premium from the stated gross wages."
-us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a $16,600 deduction, $10,600 of personal exemptions, and a 15% bracket. The actual 2026 law has a $32,200 deduction, no exemptions, and a 12% bracket. It also subtracted the $14,717 ESI premium from the $88,928 of gross wages, so it reached $5,829.15 instead of $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the 2025 OBBBA MFJ standard deduction of $31,500 instead of the 2026 inflation-adjusted $32,200. That overstated taxable income by $700 ($57,428 vs $56,727.65) and tax by about $84 ($6,395.36 vs $6,311.32)."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed tax under a TCJA sunset: a $15,850 deduction, $10,100 of personal exemptions, and a 15% second bracket. It also cut AGI to $74,211 by subtracting the $14,717 ESI premium. The correct figures are a $32,200 deduction, no exemptions, the 10%/12% brackets, and AGI of $88,927.65."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no working, and its $13,915 is more than double the correct $2,480 + 12% x $31,927.65 = $6,311.32. The number also exceeds the $10,175 tax on the full $88,928 with no deduction at all, so it applied rates far above the 2026 10%/12% brackets to the spouse's wages."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying about $27,500 of deduction plus personal exemptions and pre-TCJA brackets. It also subtracted the $14,717 ESI premium from gross wages. The correct inputs are AGI of $88,927.65, the permanent $32,200 deduction, and the 10%/12% brackets."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed a TCJA sunset with a $16,000 deduction and a 15% rate on everything above $16,800. The 2026 schedule is actually a $32,200 deduction with 10% to $24,800 and 12% above it. It also subtracted the $14,717 ESI premium from the stated gross wages."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out the spouse's $88,928 of wages. The 2026 MFJ standard deduction is only $32,200, which leaves $56,727.65 of taxable income and $6,311.32 of tax."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium to get AGI of $74,211 and then applied TCJA-sunset personal exemptions and a pre-TCJA deduction. The 2026 rules give no exemptions, a $32,200 deduction, and 10%/12% brackets on AGI of $88,927.65, for $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters ($16,100 deduction, $10,600 of exemptions, 10%/15% brackets) to an AGI of $74,211 reduced by the ESI premium. Its own inputs actually give $5,966.65, not the $6,031.65 it submitted. The correct figures are a $32,200 deduction, no exemptions, 10%/12% brackets, and AGI of $88,927.65."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 figures, a $30,000 deduction and a 10% bracket ending at $23,850, instead of the 2026 $32,200 deduction and $24,800 bracket. That gave $58,928 of taxable income and $6,594.36 instead of $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 MFJ standard deduction instead of the 2026 $32,200 and a 10% bracket ending at $24,000 instead of $24,800. That overstated taxable income at $58,928 and produced $6,591.36."
-us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It hallucinated a different household: both spouses aged 65 or older (they are 41 and 38), Form 8814 child income, and Kentucky senior and family-size credits. It computed a 5.3% state-style tax instead of the federal 10%/12% tax on $56,727.65 of taxable income, and it zeroed the result with state credits."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It showed no working. Its $10,458 is roughly the tax on the full $88,928 of wages with no deduction ($10,175 at 2026 brackets), so it effectively skipped the $32,200 MFJ standard deduction. The correct $56,727.65 of taxable income yields $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that nonrefundable credits and the standard deduction fully offset the tax. The household has no dependents or qualifying expenses, so no nonrefundable credits apply, and the $32,200 deduction leaves $56,727.65 taxable, which produces $6,311.32 of tax."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used 'scheduled 2026 post-TCJA parameters' with two personal exemptions and a pre-TCJA deduction, which gave $61,828 of taxable income. OBBBA made the $32,200 deduction, zero exemptions, and 10%/12% brackets permanent, so taxable income is $56,727.65 and tax is $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction offsets all of the $88,928 in wages. The 2026 MFJ standard deduction is $32,200, which leaves $56,727.65 of taxable income and $6,311.32 of tax."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a $17,018 deduction, $10,854 of personal exemptions, and a 15% bracket. Under OBBBA the 2026 deduction is $32,200, there are no exemptions, and the second bracket is 12%, giving $6,311.32 instead of $7,909."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset restores personal exemptions and the pre-TCJA deduction, which gave $61,928 of taxable income taxed at 10%/15%. OBBBA made the $32,200 deduction and 10%/12% brackets permanent, so taxable income is $56,727.65 and tax is $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It stated that the TCJA deduction increase and exemption suspension expire in 2026 and used a $16,600 deduction, $10,600 of exemptions, and a 15% bracket. OBBBA extended the TCJA permanently, so the correct figures are a $32,200 deduction, no exemptions, and a 12% bracket, for $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium from the stated gross wages to get AGI of $74,211 instead of $88,927.65. It also used a $30,840 deduction and a $24,520 top for the 10% bracket instead of $32,200 and $24,800, which gave $4,714 instead of $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 rules: a deduction of about $16,700, two exemptions of about $5,300 each, and a 15% bracket. The correct 2026 figures are a $32,200 deduction with no exemptions and 10%/12% brackets, so taxable income is $56,727.65 and tax is $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction plus a personal exemption reduce the spouse's $88,928 of wages to near zero. There are no personal exemptions in 2026, and the $32,200 deduction leaves $56,727.65 taxable, which produces $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It added the spouse's $18,480 of veterans benefits to AGI even though they are federally excluded. It used a $32,600 deduction and invented two children to claim a $4,000 CTC, although no dependents are listed. It then submitted -17,600, a negative value that its own $4,616.80 does not support."
-us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction reduces taxable income to zero and that CTC/ODC credits remove what remains. The $32,200 deduction leaves $56,727.65 taxable, and with no dependents listed there is no CTC or ODC, so tax is $6,311.32."
+us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Arizona offers no refundable individual credits. Arizona's refundable Credit for Increased Excise Taxes pays $25 to a single filer with AGI of $12,500 or less, and this filer's AGI is $6,736."
+us,scenario_013,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the refundable Credit for Increased Excise Taxes. With AGI of $6,736, under the $12,500 single limit, the filer gets $25 for themselves."
+us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model checked only the Family Income Tax Credit and the Property Tax Credit, which require dependents or rent/property tax. It missed the Credit for Increased Excise Taxes, which needs only AGI of $12,500 or less for a single filer and gives $25 here."
+us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model named the increased excise tax credit but treated it as needing rent or property-tax facts. The credit's only income test is AGI of $12,500 or less for a single filer. This filer's AGI is $6,736, so the $25 credit applies."
+us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model assumed the increased excise tax credit pays nothing to a single filer with no dependents and no tax liability. The credit is refundable and gives $25 for the taxpayer alone when AGI is $12,500 or less. This filer's AGI is $6,736."
+us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model found the correct credit but counted the age-65 and disability exemptions as extra $25 units, giving $75. The credit is $25 per family member, meaning the taxpayer, spouse, and dependents, so a single filer gets $25. The model also used a $25,000 single limit; the actual limit is $12,500."
+us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said Arizona's credits are nonrefundable or need dependents or property tax. It missed the refundable Credit for Increased Excise Taxes, which gives $25 to a single filer with AGI of $12,500 or less; this filer's AGI is $6,736."
+us,scenario_013,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model checked only the property tax credit and the family credit. It missed the refundable Credit for Increased Excise Taxes, which needs no rent, property tax, or dependents and gives $25 at AGI of $6,736."
+us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed that having no tax liability and no dependents rules out refundable credits. The Credit for Increased Excise Taxes is refundable and gives $25 to a single filer with AGI of $12,500 or less, regardless of liability."
+us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated dependents as required for any Arizona refundable credit. The Credit for Increased Excise Taxes gives $25 for the taxpayer alone when AGI is $12,500 or less, and this filer's AGI is $6,736."
+us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model ruled out credits because there were no dependents, donations, or property tax/rent facts. It missed the refundable Credit for Increased Excise Taxes, which depends only on AGI of $12,500 or less for a single filer and gives $25."
+us,scenario_013,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model treated the credit as requiring earned income, like an EITC. Arizona's Credit for Increased Excise Taxes has no earnings requirement and gives $25 to a single filer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the Credit for Increased Excise Taxes. A single filer with AGI of $6,736, under the $12,500 limit, receives $25."
+us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said there are no state refundable credits and missed Arizona's refundable Credit for Increased Excise Taxes. That credit gives $25 to this single filer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model said the household qualifies for no refundable credit, missing Arizona's Credit for Increased Excise Taxes. That credit gives $25 per taxpayer when single-filer AGI is $12,500 or less, and AGI here is $6,736."
+us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no Arizona refundable credit and skipped the Credit for Increased Excise Taxes. That credit gives $25 to a single filer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said Arizona has no major refundable credit this person qualifies for. It missed the Credit for Increased Excise Taxes, which gives $25 to a single filer with AGI of $12,500 or less."
+us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds the credit limits. Federal AGI is $6,736 because none of the Social Security is taxable, which is well under the $12,500 single limit for the Credit for Increased Excise Taxes, so the $25 credit applies."
+us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the Credit for Increased Excise Taxes. That credit gives $25 to a single filer with AGI of $6,736, under the $12,500 limit."
+us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said no refundable credits apply and missed Arizona's Credit for Increased Excise Taxes. That credit gives $25 to this single, low-AGI filer."
+us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said nothing in the facts triggers a refundable credit, missing Arizona's Credit for Increased Excise Taxes. The facts given, a single filer with AGI of $6,736, are enough to qualify for $25."
+us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model looked for dependents or earned income. Arizona's Credit for Increased Excise Taxes needs neither and gives $25 to a single filer with AGI of $12,500 or less; AGI here is $6,736."
+us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model found no supported Arizona refundable credit and missed the Credit for Increased Excise Taxes. With $6,736 AGI, a single filer receives $25."
+us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model required property tax, rent, or dependents. Arizona's refundable Credit for Increased Excise Taxes depends only on income (AGI of $12,500 or less for a single filer) and gives $25 here."
+us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model concluded a one-person elderly household gets no Arizona refundable credit. The Credit for Increased Excise Taxes gives $25 to a single taxpayer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no refundable Arizona credit is supported, missing the Credit for Increased Excise Taxes. AGI of $6,736 is under the $12,500 single limit, so the credit is $25."
+us,scenario_013,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model required earnings, dependents, or expenses. Arizona's Credit for Increased Excise Taxes needs none of these and gives $25 to a single filer with AGI of $12,500 or less."
+us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model looked at the Working Poor Tax Credit, which is a nonrefundable donation credit, and missed the refundable Credit for Increased Excise Taxes. That credit gives $25 to this single filer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model checked only the property tax credit and missed the Credit for Increased Excise Taxes. That credit needs no property tax or other expenses and gives $25 at AGI of $6,736."
+us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said household composition and income rule out refundable credits. A single filer with AGI of $6,736 meets the $12,500 limit for Arizona's Credit for Increased Excise Taxes, which pays $25."
+us,scenario_013,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model treated zero taxable income and no dependents as disqualifying. The Credit for Increased Excise Taxes is refundable and gives $25 for the taxpayer alone when AGI is $12,500 or less."
+us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model looked only at EITC-style and property tax credits. Arizona's Credit for Increased Excise Taxes has no earned-income or expense requirement and gives $25 at AGI of $6,736."
+us,scenario_013,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model tested $30,544 of gross household income, including non-taxable Social Security, against a $25,000 single limit. The credit tests federal AGI, which is $6,736 because no Social Security is taxable here, against a $12,500 single limit, so the $25 credit applies."
+us,scenario_013,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model treated dependents as required for Arizona refundable credits. The Credit for Increased Excise Taxes gives $25 for the taxpayer when single-filer AGI is $12,500 or less."
+us,scenario_013,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model applied the Property Tax Credit's income limit and ignored the Credit for Increased Excise Taxes, a separate credit. That credit tests AGI ($6,736 here) against $12,500 and gives $25."
+us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model required qualifying children or earned income. Arizona's Credit for Increased Excise Taxes needs neither and gives $25 to a single filer with AGI of $6,736."
+us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model assumed a refundable credit needs tax liability to offset. Arizona's Credit for Increased Excise Taxes is refundable and gives $25 to a single filer with AGI of $12,500 or less, even with zero liability."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premiums from the gross wage input, which cut AGI to $74,211 instead of $88,927.65. It then submitted $5,647, which matches neither its own $4,545 calculation nor the correct $56,727.65 × 2026 bracket result of $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $28,000 standard deduction instead of the 2026 MFJ amount of $32,200. It also taxed all of its taxable income at 12% and skipped the 10% bracket on the first $24,800. Finally, it replaced its own $7,311 result with an unexplained $6,819."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model gave a rough '12% bracket' estimate of $6,900 and never actually ran the brackets. The correct calculation subtracts the $32,200 standard deduction, takes 10% of the first $24,800, and takes 12% of the remaining $31,927.65, for $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an outdated 10% bracket ceiling of $24,150 instead of the 2026 MFJ figure of $24,800, which gave $6,324. It then dropped even that figure for an unsupported $5,235."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own bracket calculation reached about $6,311, essentially the correct answer, but it submitted $7,466. Nothing in its reasoning supports that number."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model applied the correct $32,200 standard deduction and the $24,800 10% bracket ceiling and arrived at $6,311. It then replaced that with an unsupported $6,712, citing 'estimated bracket thresholds'."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 values: a $30,000 standard deduction and a $23,850 10% bracket ceiling. The 2026 MFJ values are a $32,200 deduction and a $24,800 ceiling, so the model overstated taxable income by $2,200 and produced $6,594."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model guessed a $32,600 standard deduction and a $24,000 10% bracket ceiling, which gave $6,279. It then submitted an unsupported $8,079 instead of the correct $6,311.32 from the $32,200 deduction and $24,800 ceiling."
+us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $30,600 standard deduction and a $24,400 10% bracket ceiling. The correct 2026 MFJ values are $32,200 and $24,800, so its taxable income was $1,600 too high and its tax came to $6,511."
+us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA expired in 2026 and applied a $15,400 standard deduction, personal exemptions, and a 15% bracket. OBBBA made the $32,200 MFJ deduction and the 10%/12% brackets permanent. The model also subtracted the $14,717 ESI premiums from the gross wage input."
+us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied TCJA-sunset rules: a $16,600 standard deduction, $10,600 in personal exemptions, and a 15% second bracket. Under OBBBA, 2026 law has a $32,200 deduction, no exemptions, and a 12% bracket. The model also wrongly cut wages by the $14,717 ESI premiums."
+us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used the 2025 OBBBA standard deduction of $31,500 instead of the inflation-adjusted 2026 MFJ amount of $32,200. That overstated taxable income by $700 and tax by $84, giving $6,395.36."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA sunset and applied a $15,850 standard deduction, $10,100 in exemptions, and a 15% bracket. 2026 law instead has a $32,200 deduction and 10%/12% brackets. The model also subtracted the $14,717 ESI premiums, which reduced AGI to $74,211."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $13,915 is more than double the correct tax and is consistent with a much smaller deduction or higher rates. The correct calculation is $88,927.65 minus the $32,200 deduction, giving $56,727.65 of taxable income, with only 10% and 12% brackets applying, for $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired and used pre-TCJA deductions, exemptions totaling about $27,500, and pre-TCJA brackets. It also subtracted the $14,717 ESI premiums from wages. In fact, 2026 law has a $32,200 standard deduction and 10%/12% brackets applied to $88,927.65 of AGI."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and used a $16,000 standard deduction, a 10% bracket of only $16,800, and a 15% rate. It also subtracted the ESI premiums from wages. The 2026 MFJ values are a $32,200 deduction, a $24,800 10% bracket, and a 12% rate on wages of $88,927.65."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction reduces tax to zero, but $32,200 leaves $56,727.65 of taxable income from the $88,927.65 in wages. That produces $6,311.32 in tax, and no nonrefundable credits apply to offset it."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model subtracted the $14,717 ESI premiums, which reduced AGI to $74,211. It also applied personal exemptions and TCJA-sunset rules, even though OBBBA set 2026 law at a $32,200 standard deduction with no exemptions and 10%/12% brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied TCJA-sunset rules: a $16,100 standard deduction, $10,600 in exemptions, and a 15% bracket above $23,200. It also subtracted the $14,717 ESI premiums. 2026 law instead has a $32,200 deduction and 10%/12% brackets applied to $88,927.65 of wages."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 values: a $30,000 standard deduction and a $23,850 10% bracket ceiling. The correct 2026 MFJ values are $32,200 and $24,800, so its taxable income was $2,200 too high and its result was $6,594.36."
+us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $30,000 standard deduction and a $24,000 10% bracket ceiling instead of the 2026 MFJ values of $32,200 and $24,800. As a result, it computed tax on $58,928 rather than $56,727.65."
+us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"The model invented facts that are not in the household data: both spouses over 65, a Kentucky rate of 5.3%, Form 8814 child income, and state family-size and elderly credits. It never computed federal tax on the $56,727.65 of taxable income left after the $32,200 MFJ standard deduction."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's $10,458 is far above the correct figure and is consistent with too small a deduction or too high a marginal rate. The correct result applies the 2026 $32,200 MFJ deduction to $88,927.65 of wages, then 10% up to $24,800 and 12% on the remainder, for $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model claimed nonrefundable credits or the standard deduction eliminate the tax. The household has no dependents or qualifying expenses, so there are no nonrefundable credits, and $56,727.65 of taxable income remains after the $32,200 deduction."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied 'scheduled post-TCJA' parameters: a smaller standard deduction plus two personal exemptions, which gave about $61,828 of taxable income, along with a 15% bracket. OBBBA made the $32,200 MFJ deduction and the 10%/12% brackets permanent for 2026."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction offsets all $88,928 of wages. The 2026 MFJ deduction is only $32,200, which leaves $56,727.65 of taxable income and $6,311.32 of tax."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and used a $17,018 standard deduction, $10,854 in personal exemptions, and a 15% second bracket. 2026 law under OBBBA has a $32,200 deduction, no exemptions, and a 12% bracket up to $100,800."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model restored personal exemptions and the pre-TCJA standard deduction and taxed $61,928 at 10%/15% rates. OBBBA made the TCJA structure permanent, so the correct calculation uses the $32,200 deduction and 10%/12% brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model treated 2026 as a TCJA-sunset year, with a $16,600 standard deduction, $10,600 in exemptions, and a 15% bracket. Under OBBBA, the MFJ deduction is $32,200, there are no exemptions, and the second bracket is 12%."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premiums from the gross wage input, which cut AGI to $74,211 instead of $88,927.65. It also used an understated $30,840 standard deduction and a $24,520 10% bracket instead of the 2026 values of $32,200 and $24,800."
+us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied pre-TCJA rules: a standard deduction of about $16,700, two exemptions of about $5,300 each, and a 15% bracket. 2026 law has a $32,200 MFJ deduction with 10%/12% brackets, which yields $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction and a personal exemption reduce the wages to near zero. 2026 law has no personal exemption, and the $32,200 MFJ deduction leaves $56,727.65 of taxable income and $6,311.32 of tax."
+us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model included the spouse's $18,480 of veterans benefits in AGI, even though VA benefits are fully excluded. It also used a $32,600 deduction and invented two children to claim a $4,000 CTC, then submitted a negative $-17,600 that contradicts its own $4,616.80."
+us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied CTC/ODC to wipe out the tax, but the household has no children or other dependents, so neither credit exists. It also wrongly claimed the standard deduction zeroes out $88,927.65 of wages."
us,scenario_014,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract.
us,scenario_014,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded refundable CTC despite the household having no qualifying children and awarded EITC despite approximately $183,584 of joint income, which exceeds the childless EITC phase-out limit. Those two credits both equal zero, so their asserted $3,067 total has no eligible credit pathway."
us,scenario_014,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the head's lack of earned income as though Medicaid were tested on the head's income alone, omitting the spouse's wages and other amounts included in household tax-unit MAGI. The resulting MAGI is 4.11 times FPL, above the West Virginia adult limit, and the head has no pregnancy, disability, dependent-child, SSI, or other categorical eligibility pathway."
@@ -1127,87 +1143,89 @@ us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there is no answer to grade. The correct derivation is 7.65% of the spouse's $88,928 wages, about $6,803."
us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model's explanation computes $5,513.54 Social Security plus $1,289.46 Medicare, about $6,803, on the spouse's $88,928 wages. It then submitted 'value = 6581', a number that contradicts its own sum."
us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct components, $5,513.54 Social Security and $1,289.46 Medicare on $88,928 with no Additional Medicare Tax. It then submitted $8,075.02 instead of their $6,802.99 sum, adding about $1,272 that its own explanation never supports."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the right $84,928 base but applied the 2025 rates (2.22–4.82%) instead of the 2026 schedule (2.11/2.81/3.16/4.22/4.58%). It then raised its own ~$3,256 result to $3,374 with no basis."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rates (2.13/2.84/3.20/4.26/4.63%) reflect about a 4% cut from 2025. The 2026 cut is 5% (2.11/2.81/3.16/4.22/4.58%), so it overstated tax by about $33."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium from gross wages and skipped the $4,000 personal exemptions. It then applied a guessed ~4.5% average rate instead of the graduated 2026 MFJ brackets on $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got $3,255 using the 2025 rates instead of the 5%-lower 2026 schedule. It then subtracted about $378 of nonexistent nonrefundable credits to reach $2,877."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The $84,928 base is right, but it mixed 2025 and made-up rates (4.07%, 4.625%) to get about $3,130. It then applied an unexplained extra cut down to $2,849 instead of the 2026 rates of 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The $84,928 base is right, but $1,655 is only about a 1.95% average rate. The 2026 brackets produce $3,092.19, so it greatly understated the rates and assumed nonrefundable credits that do not exist."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22/2.96/3.33/4.44/4.82%) to the correct $84,928 and missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2023 21.25%-cut rates with a collapsed four-bracket structure to get $3,360. It then cut the answer to $1,919 for credits it admitted do not apply, instead of using the 2026 five-bracket schedule."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The $84,928 base is right, but it guessed an approximate $3,200 from vague 2.36–5.12% rates. It never applied the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,It assumed the 2023–24 rates (2.36/3.15/3.54/4.72/5.12%) still apply in 2026. That misses both the 2025 cut and the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%.
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a $4,000 WV standard deduction and $6,000 exemptions per person. WV has no standard deduction and allows $2,000 per exemption, so taxable income is $84,927.65. It also used a made-up rate schedule."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium, getting AGI of $74,211 instead of $88,928, and invented a $15,400 WV standard deduction instead of $4,000 in exemptions. It also used wrong rates (2.50/3.13/5.12%)."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $74,211 by subtracting the ESI premium and added a $4,000 standard deduction that WV does not have. Its rates (2.18–4.72%) reflect only about a 2% cut instead of the 2026 5% cut."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $8,000 when WV allows only the $4,000 of exemptions, leaving $80,928 instead of $84,927.65. Its rates (1.69–4.29%) are far below the 2026 schedule of 2.11–4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium, getting a $70,211 base instead of $84,927.65. It also applied the outdated 2.36–5.12% rates instead of the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. $4,125 matches a flat ~4.6% rate on gross wages with no $4,000 exemption and no graduated brackets, but the 2026 brackets on $84,927.65 yield $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $70,211 base after wrongly subtracting the ESI premium from gross wages. It also applied the old 2.36–5.12% rates instead of the 5%-reduced 2026 rates on $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium, getting a $70,211 base instead of $84,927.65. It then used the outdated 2.36–5.12% schedule instead of the 2026 rates of 2.11–4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It assumed standard deductions and exemptions wipe out the tax. WV has no standard deduction and allows only $4,000 of exemptions, leaving $84,927.65 taxable and $3,092.19 of tax."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It reduced the base to $70,211 by subtracting the ESI premium from gross wages. It also applied the 2023–24 rates (2.36–5.12%) instead of the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used a $70,211 base after subtracting the ESI premium, instead of $84,927.65. It also applied an outdated, overstated rate schedule instead of the 2026 5%-reduced brackets."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $84,928 base is right, but it applied the 2023–24 rates (2.36/3.15/3.54/4.72/5.12%). It missed the 2025 and 2026 cuts, which bring the rates to 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It invented an $8,000 WV standard deduction and taxed all of the remaining $76,928 at a flat 4.82% top rate. It should have used the graduated 2026 MFJ brackets on $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It computed Kentucky tax on an invented $25,050 of income with a family size credit, even though the household is in West Virginia. WV tax on $84,927.65 at 2026 rates is $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation. $2,331 is about a 2.7% average rate on $84,928, well below what the 2026 brackets (2.11–4.58%) produce on that base, which is $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and nonrefundable items fully eliminate WV tax. WV allows only $4,000 of exemptions and no standard deduction, and no nonrefundable credit applies at this income."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22/2.96/3.33/4.44/4.82%) to the correct $84,928 and missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rate schedule overstates the 2026 rates by about 4%, giving $3,217.53. The 5%-reduced schedule (2.11–4.58%) gives $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It carried the 2025 rates into 2026, getting $3,255.03. It missed the 5% cut to 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"$3,255 is the 2025 schedule applied to $84,928. It missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22–4.82%) instead of the 2026 schedule (2.11/2.81/3.16/4.22/4.58%) to the correct $84,928 base."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rate schedule reflects only about a 3% cut from 2025 instead of the 5% 2026 cut, so it overstated tax by about $100."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,It used the 2025 joint rates of 2.22/2.96/3.33/4.44/4.82% instead of the 5%-lower 2026 rates of 2.11/2.81/3.16/4.22/4.58%.
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The $84,928 base is right, but $2,952.20 matches about a 9–10% cut from the 2025 rates. The 2026 cut is 5% (2.11/2.81/3.16/4.22/4.58%), so it understated tax."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It assumed a standard deduction wipes out WV tax. WV has no standard deduction, only $2,000 per exemption, so $84,927.65 stays taxable."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,It assumed the 2024 rates (2.36–5.12%) remain the baseline for 2026. It missed the 2025 cut and the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%.
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted an invented ~$20,600 of deductions when WV allows only $4,000 of exemptions. It also used the outdated 2.36–5.12% rates."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,It carried rates of 2.27/3.02/3.40/4.53/4.92% into 2026. These are higher than both the 2025 schedule and the 2026 schedule (2.11/2.81/3.16/4.22/4.58%).
-us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium to get AGI of $74,211, took off the $30,840 federal standard deduction, which WV does not allow, and used made-up 2/3/4/5% rates. The correct figures are $4,000 of exemptions and the 2026 WV rate schedule."
-us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ~$84,900 base is right, but $3,422 matches the pre-2025 rate schedule. It missed the reductions to 2.11/2.81/3.16/4.22/4.58% in 2026."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium from gross wages and invented an $8,000 WV standard deduction. It then applied a nonexistent flat 4.5% rate instead of the graduated 2026 brackets on $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the 2024 rates, $2,183.50 plus 5.12% over $60k, to $84,928. It missed the cuts to a 4.58% top rate and to 2.11/2.81/3.16/4.22% below it."
-us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the personal exemption reduces taxable wages to zero. WV exemptions total only $4,000 for two people, leaving $84,927.65 taxable."
-us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The $84,928 base is right, but it used inconsistent bracket amounts and a 4.82% top rate from the 2025 schedule. The 2026 rates are 2.11/2.81/3.16/4.22/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It subtracted the ESI premium, getting a $70,211 base, and applied an invented, inverted rate schedule that declines as income rises. It also subtracted a nonexistent 4% family tax credit instead of applying the 2026 progressive brackets."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the federal standard deduction and assumed nonrefundable credits eliminate the tax. WV allows neither; the only offset is the $4,000 of exemptions, leaving $3,092.19 of tax."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It derived the correct AGI of $42,909, taxable income of $26,809, and tax of $1,240 + $1,729 = $2,969. It also correctly ruled out the Saver's Credit. It then submitted $3,103 with no computation behind the $134 increase, overriding its own correct result."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used 2024 parameters (a $14,600 standard deduction and a $11,600 top for the 10% bracket) instead of the 2026 values of $16,100 and $12,400. It also included the $960 of nontaxable workers' compensation and deducted the full $4,510 capital loss instead of the $3,000 cap. It then submitted $3,509, which does not match its own $3,094 computation."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly reached AGI of $42,909 and taxable income of $26,809. It then used a $12,150 top for the 10% bracket instead of the 2026 value of $12,400, which gave $2,974. It submitted $3,268 anyway, with no step supporting the extra $294."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said the $1,042 traditional 401(k) deferral was already excluded from wages, but then built AGI from the full $45,000. That gave AGI of $43,951 instead of $42,909 and taxable income of $27,851 instead of $26,809. The $3,094 result is exactly $125 (12% of $1,042) above the correct $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,510 capital loss, wrongly saying it was under the limit, instead of capping the deduction at $3,000. It never excluded the $1,042 traditional 401(k) deferral, and it used a $16,700 standard deduction instead of $16,100. It then submitted $4,172, which contradicts its own $2,841 computation."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"It correctly found taxable income of $26,809 and the $16,100 standard deduction. It reported only the 12% slice (12% × $14,409 = $1,729.08) and dropped the $1,240 of tax on the first $12,400 at 10%, so the correct $2,969.13 was never summed."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the correct AGI of $42,909, then chose 2025 parameters as a proxy: a $15,000 standard deduction and a $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400. That inflated taxable income to $27,909 and tax to $3,111."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It used a $15,750 standard deduction instead of the 2026 value of $16,100, which gave $3,011. It then subtracted a $135 Saver's Credit even though AGI of $42,909 is above the single Saver's Credit income limit. On top of that it cut the result to $1,750 with no identifiable credit or deduction behind the reduction."
-us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized using $13,800 of mortgage interest invented as 6% of the loan balance, although no mortgage interest was given and unlisted amounts are 0. It also double-counted the $2,080 premium to reach $10,200 of medical expenses. Legitimate itemized deductions (medical above 7.5% of AGI plus Indiana tax) fall well short of the $16,100 standard deduction, so taxable income is $26,809, not $20,891."
-us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It treated the $2,080 health insurance premium as a pre-tax wage reduction, but the premium is not a payroll exclusion here, so AGI stays at $42,909 rather than $40,829. It also used an outdated $15,000 standard deduction and $11,600 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400."
-us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law for 2026: an $8,250 standard deduction, a $5,250 personal exemption, and a 15% second bracket. The TCJA structure is permanent, so 2026 uses a $16,100 standard deduction, no personal exemption, and 10%/12% brackets. That puts taxable income at $26,809, not $29,409."
-us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health insurance premium as a pre-tax wage exclusion, which lowered AGI to $40,829 instead of $42,909. It also used a $15,750 standard deduction instead of $16,100. The resulting $25,079 of taxable income is below the correct $26,809."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and used an $8,300 standard deduction, a $5,050 personal exemption, and a 15% bracket. The 2026 schedule actually has a $16,100 standard deduction, no exemption, and 10%/12% brackets with the 10% bracket ending at $12,400. That inflated taxable income to $29,559 and tax to $3,853.85."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It started from AGI of about $44,000, which omits the $1,042 401(k) exclusion and $49 IRA deduction, and then subtracted nonrefundable credits even though none apply. With the $16,100 standard deduction, the correct taxable income of $26,809 produces $2,969.13. The $1,541 answer can only come from an unsupported credit or deduction."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied a pre-2018 standard deduction plus personal exemption with 15% brackets, instead of the 2026 $16,100 standard deduction and 10%/12% schedule. It also wrongly excluded the $2,080 health insurance premium from wages."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used a $15,000 standard deduction and a $11,600 top for the 10% bracket. The 2026 values are $16,100 and $12,400, which give taxable income of $26,809 and tax of $2,969.13 rather than $3,117."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its computation lists only wages, interest and the $3,000 capital-loss limit. It omitted the $1,042 traditional 401(k) exclusion and the $49 IRA deduction that bring AGI down to $42,909. It never showed the 2026 figures of a $16,100 standard deduction, $26,809 taxable income, and $2,969.13 tax."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health insurance premium as a pre-tax wage exclusion, dropping AGI to $40,829 from the correct $42,909. It then used a $15,700 standard deduction and a 2025 bracket top of $11,925 instead of the 2026 values of $16,100 and $12,400."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then applied a standard deduction plus personal exemption ""under 2026 law"" as if the TCJA had sunset. The 2026 schedule has no personal exemption, a $16,100 standard deduction, and 10%/12% brackets, which give $2,969.13 rather than $3,788.85."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, but its $3,111 matches the 2025 parameters: a $15,000 standard deduction and a $11,925 top for the 10% bracket (taxable income of $27,909). The 2026 values are $16,100 and $12,400, which give $26,809 of taxable income."
-us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used an estimated $15,400 standard deduction and a $12,247 top for the 10% bracket. The 2026 values are $16,100 and $12,400, so taxable income was overstated at $27,509."
-us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It had the correct AGI and the correct 10% bracket top of $12,400, but used the 2025 standard deduction of $15,750 instead of the 2026 value of $16,100. That overstated taxable income by $350 ($27,159) and tax by $42."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It assumed nonrefundable credits wipe out the tax, but none are available. There are no dependents, the Saver's Credit is gone because AGI of $42,909 exceeds the single limit, and the elderly/disabled credit's $5,000 base is fully eliminated by the reduction of half of AGI over $7,500. The $26,809 of taxable income therefore carries $2,969.13 of tax."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated child support paid as deductible, which it is not. Its $5,199 is higher than the tax on the entire $42,909 AGI with no standard deduction at all (about $4,901). So its number implies it never subtracted the $16,100 standard deduction that brings taxable income down to $26,809."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"It correctly found taxable income of $26,809, taxed $12,400 at 10% ($1,240) and $14,409 at 12% ($1,729.08), and applied no credits. It then added those two pieces wrong, reporting $3,211.08 instead of $2,969.08."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It deducted all $1,351 of retirement contributions, including the $184 Roth 401(k) and $76 Roth IRA contributions, which are after-tax. It also subtracted the full $4,510 capital loss instead of the $3,000 cap. Its $3,200 does not follow from the correct $26,809 taxable income under the 2026 $16,100 standard deduction and 10%/12% brackets."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then applied pre-TCJA rules for 2026: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The 2026 law uses a $16,100 standard deduction with no exemption and 10%/12% brackets, which give $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, but its taxable income of $27,159 implies the 2025 standard deduction of $15,750 rather than the 2026 value of $16,100. That overstated taxable income by $350 and gave $3,011 instead of $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used an estimated $15,400 standard deduction and a $12,250 top for the 10% bracket. The 2026 values are $16,100 and $12,400, so taxable income came out at $27,509 instead of $26,809."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset and inflated pre-2018 law, using about $14,131 of standard deduction plus exemption and a 15% bracket above $12,670. The 2026 schedule keeps the $16,100 standard deduction and 10%/12% brackets, so taxable income is $26,809 and tax is $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized with about $13,800 of mortgage interest invented from the $230,000 loan balance, although no mortgage interest was listed and unlisted amounts are 0. It also double-counted the $2,080 premium in its medical deduction. The genuine itemizable amounts fall short of the $16,100 standard deduction, so taxable income is $26,809, not $20,891."
-us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $2,969.13 reference."
-us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It assumed the standard deduction plus credits tied to the disability would zero out the tax. The elderly/disabled credit's $5,000 base is fully eliminated by the reduction of half of AGI over $7,500, and no other nonrefundable credit applies, so $26,809 of taxable income carries $2,969.13 of tax."
-us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $960 of workers' compensation to AGI even though that income is excluded, and it omitted the $49 IRA deduction. It also used a $15,750 standard deduction and a $12,150 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. It then submitted $2,104, which contradicts its own $3,137.16 computation."
-us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It computed tax on MFJ brackets for a single filer with no spouse, and subtracted a $500 credit for other dependents and an invented child-related credit even though there are no dependents. It also never excluded the $1,042 401(k) deferral or the $49 IRA deduction. The correct single computation is $26,809 of taxable income and $2,969.13 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the $84,928 base right and computed about $3,256 using the 2025 WV rates. It then raised the figure for no stated reason to $3,374. The 2026 schedule is about 5% below the 2025 rates, so the tax is $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The taxable income of $84,928 is correct. The model's 2026 rates (2.13/2.84/3.20/4.26/4.63%) are only about 4% below the 2025 schedule. The actual 2026 cut is 5% (about 2.11/2.81/3.16/4.22/4.58%), so it overstated the tax by about $33."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, which does not reduce AGI here. It also skipped the two $2,000 WV personal exemptions and applied a made-up ~4.5% average rate instead of the graduated 2026 MFJ brackets on $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model gave no derivation and a rough $2,700. The 2026 graduated MFJ brackets (about 2.11% to 4.58%) applied to $88,928 AGI minus $4,000 in exemptions produce $3,092. The model understated that by about $390."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model computed $3,255 on the correct $84,928 base using the 2025 rates. It then cut that to $2,877 for unnamed nonrefundable credits, even though none apply. It never applied the 2026 rates, which are another 5% lower and give $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The base is correct, but the model mixed 2025-era rates with invented ones (4.07%, 4.625%) to reach about $3,130. It then applied an arbitrary 'further cut' down to $2,849 instead of using the 2026 schedule, which yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income, but $1,655 is only about half of the tax the 2026 graduated brackets produce on that base ($3,092). It also cited nonrefundable credits, but none apply."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model applied the 2025 WV rates (2.22% to 4.82%) to the correct $84,928 base. It missed the further 5% cut for 2026, which lowers the brackets to about 2.11% to 4.58% and the tax to $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an outdated four-bracket schedule (2.36/3.15/3.54/4.72%) to get about $3,360. It then dropped to $1,919 by assuming credits that it had itself just said do not apply. The 2026 five-bracket schedule on $84,928 gives $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but used a 2023–2025 rate range topping out at 4.82%, then rounded to $3,200. It missed the 2026 schedule, which tops out at about 4.58% and yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model assumed the 2024 brackets (2.36% to 5.12%) still apply in 2026. Two rounds of rate cuts have lowered them to about 2.11% to 4.58%, so the tax on $84,928 is $3,092, not $3,460."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model invented a $4,000 WV standard deduction, which does not exist, and $6,000 personal exemptions per person, when the exemption is $2,000 each. That cut taxable income to $72,928 instead of $84,927.65. It also used made-up brackets rather than the 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, giving an AGI of $74,211 instead of $88,928. It then took a $15,400 standard deduction, which WV does not allow, skipped the $2,000 personal exemptions, and applied invented rates up to 5.12%."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The rates it used are close to the 2026 schedule. But the model cut the base by subtracting the $14,717 ESI premium (AGI $74,211) and adding a nonexistent $4,000 WV standard deduction, which left $66,211 instead of $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model subtracted $8,000 instead of the correct $4,000 in personal exemptions (WV has no standard deduction). It also applied made-up rates from 1.69% to 4.29%, well below the 2026 schedule of about 2.11% to 4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium, giving an AGI of $74,211 instead of $88,928, so taxable income was $70,211 instead of $84,927.65. It also applied the 2024 brackets (2.36% to 5.12%) rather than the lower 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no derivation. $4,125 is about 4.86% of the full $84,928, which is a flat top-bracket-style rate. WV taxes this income with graduated 2026 brackets (about 2.11% to 4.58%), producing $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $14,717 ESI premium as a reduction to wages, giving $74,211 of AGI and $70,211 of taxable income instead of $88,928 and $84,927.65. It then applied the 2024 rates instead of the 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, giving an AGI of $74,211 and taxable income of $70,211 instead of $84,927.65. Its result also matches the 2024 brackets rather than the 2026 rates, which are about 5% below the 2025 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model claimed deductions and exemptions wipe out the liability. WV has no standard deduction, and the only subtraction is $4,000 of personal exemptions, which leaves $84,927.65 taxable and $3,092 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI by the $14,717 ESI premium, giving $74,211 instead of $88,928, and applied the 2024 brackets to $70,211. The correct computation applies the 2026 schedule to $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $74,211 by subtracting the $14,717 ESI premium, which understated taxable income by $14,717. It then applied an unspecified rate schedule that is higher than the 2026 brackets."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but applied the 2024 brackets (2.36/3.15/3.54/4.72/5.12%). It missed the 2025 cut and the further 5% 2026 cut that bring the tax to $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 WV standard deduction, which does not exist. It then applied the 4.82% top rate flat to $76,928 instead of running $84,927.65 through the graduated 2026 brackets."
+us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model computed Kentucky income tax and a Kentucky family size credit for a West Virginia household. The WV computation ($88,928 AGI minus $4,000 in exemptions, run through the 2026 MFJ brackets) yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no derivation. $2,331 is about 75% of the correct liability, which is consistent with a reduced base or rates far below the 2026 WV MFJ schedule applied to $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits eliminate the WV tax. WV allows only $4,000 of personal exemptions for this couple, and no nonrefundable credit applies at $88,928 of AGI, so the tax is $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but applied the 2025 brackets (2.22% to 4.82%) as if they were the 2026 schedule. The 2026 rates are another 5% lower, which reduces the tax from $3,255 to $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The base is correct ($84,928). The model's rate schedule produces $3,217.53, only about 1% below the 2025 schedule. The full 5% 2026 cut gives $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied the 2025 WV rates to the correct $84,928, getting $3,255.03. It missed the 2026 cut of about 5%, which lowers every bracket and brings the tax to $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $3,255 is the 2025 rate schedule applied to the correct $84,928 base. It did not apply the further 5% rate reduction for 2026, which yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the 2025 brackets (2.22/2.96/3.33/4.44/4.82%) for tax year 2026. The 2026 brackets are 5% lower (about 2.11/2.81/3.16/4.22/4.58%), so the tax on $84,928 is $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The base is correct ($84,928), but the model's schedule is only about 2% below the 2025 rates. The 2026 brackets are about 5% below them (roughly 2.11% to 4.58%), which yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The taxable income is correct, but the model applied the 2025 joint rates (2.22% to 4.82%). It missed the 2026 reduction to about 2.11% to 4.58%, which yields $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The base is correct ($84,928), but the model's 2026 rates are about 4.5% below the real 2026 brackets. That understates the tax by about $140 compared with the schedule that is 5% below the 2025 rates, which gives $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction that WV does not have and concluded the tax is zero. WV subtracts only $2,000 per exemption, which leaves $84,927.65 taxable and $3,092 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed the 2024 brackets (2.36% to 5.12%) are still the 2026 baseline. The 2025 and 2026 cuts lower them to about 2.11% to 4.58%, so the tax on $84,928 is $3,092, not $3,460."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted about $20,600 of made-up deductions and exemptions, when WV allows only $4,000 of personal exemptions and no standard deduction. It also applied the 2024 rates (2.36% to 5.12%) instead of the 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model assumed its estimated 2025 rates (2.27% to 4.92%) carry into 2026. Those are already above the actual 2025 rates, and the 2026 brackets are another 5% lower (about 2.11% to 4.58%), giving $3,092 on $84,928."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from AGI and then a $30,840 federal standard deduction, which WV does not allow. WV starts from federal AGI and subtracts only $2,000 per exemption. The model also used outdated 2%–5% rates."
+us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The base is correct (about $84,900), but $3,422 matches the higher pre-cut 2024-era rates. The 2026 brackets (about 2.11% to 4.58%) yield $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium (AGI $74,211) and an $8,000 WV standard deduction that does not exist. It then applied a nonexistent flat 4.5% rate instead of the graduated 2026 MFJ brackets on $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used the 2024 schedule ($2,183.50 plus 5.12% above $60,000) on the correct $84,928. It missed the 2025 and 2026 rate reductions that bring the top rate to about 4.58% and the tax to $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed the personal exemption reduces taxable wages to zero. The WV exemption is only $2,000 per person ($4,000 total), which leaves $84,927.65 taxable and $3,092 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but mixed in 2025 rates (including the 4.82% top rate) and inconsistent bracket amounts. The 2026 brackets, 5% below 2025, yield $3,092."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium to get $74,211 of AGI and built an inverted bracket schedule with rates falling as income rises. It also applied a 4% 'family tax credit' that does not apply at this income. The correct computation runs $84,927.65 through the ascending 2026 MFJ brackets."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the federal standard deduction and assumed nonrefundable credits eliminate the tax. WV allows no standard deduction, only $4,000 of exemptions, and no credit applies here, so $84,927.65 is taxed for $3,092."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference exactly: AGI $42,909, taxable income $26,809, and tax of about $2,969 at 10% to $12,400 and 12% above. It then submitted $3,103 with no derivation. That $134 overstatement is a transcription error, not a different rule."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It counted the $960 of nontaxable workers' compensation as income and deducted the full $4,510 capital loss instead of the $3,000 cap. It also used the 2024 single standard deduction ($14,600) and 2024 10% bracket ($11,600) instead of the 2026 values ($16,100 and $12,400). It then dropped its own $3,094 result and submitted an unexplained $3,509."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It applied 'preferential capital gain treatment' even though the net capital result is a loss, capped at $3,000 against ordinary income, so every dollar of the $26,809 taxable income is taxed at ordinary rates. Applying 10%/12% to that amount gives $2,969. The $2,900 answer reflects understated income or misapplied rates."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct $26,809 taxable income but put the 10% bracket top at $12,150 instead of the 2026 value of $12,400, which gave $2,974. It then submitted an unexplained $3,268 that does not follow from its own calculation."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said the $1,042 traditional 401(k) deferral was already excluded from wages but still started from the full $45,000. That left AGI at $43,951 instead of $42,909 and taxable income at $27,851 instead of $26,809, which overstates tax by about $125."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the full $4,510 capital loss as deductible, wrongly calling it 'under the $3,000 limit'. It also left the $1,042 401(k) deferral in wages and used a $16,700 standard deduction instead of $16,100. It then threw out its own $2,841 calculation and submitted an unexplained $4,172."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"It reached the correct $26,809 taxable income but reported only the 12% bracket piece ($14,409 × 12% = $1,729.08). It left out the $1,240 of tax on the first $12,400 in the 10% bracket, which brings the total to $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the correct $42,909 AGI but used 2025 parameters for 2026: a $15,000 standard deduction instead of $16,100 and a $11,925 10% bracket top instead of $12,400. That produced taxable income of $27,909 and tax of $3,111."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It used the 2025 $15,750 standard deduction instead of $16,100. It also applied a Saver's Credit even though AGI of $42,909 is above the single Saver's Credit AGI limit, and it counted Roth contributions toward that credit. It then made further unexplained 'adjustments' down to $1,750, which no rule supports."
+us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized using $13,800 of mortgage interest it made up from the loan balance (no interest paid was listed, so it is $0). It also counted the $2,080 premium twice in medical expenses and estimated a SALT amount. With no mortgage interest, itemized deductions fall below the $16,100 standard deduction, so taxable income is $26,809, not $20,891. It also used the 2025 $11,925 10% bracket."
+us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health premium from wages as a pre-tax deduction, but the facts give $45,000 as gross wages and nothing says the premium was paid pre-tax, so AGI is $42,909. It also used a $15,000 standard deduction and the 2024 $11,600 10% bracket instead of the 2026 $16,100 and $12,400."
+us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had expired for 2026 and applied an $8,250 standard deduction, a $5,250 personal exemption and a 15% bracket. The OBBBA kept the TCJA structure, so 2026 has a $16,100 standard deduction, no exemption, and 10%/12% brackets that give $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health premium from wages as pre-tax, which the facts do not support, so it understated AGI at $40,829 instead of $42,909. It also used the 2025 $15,750 standard deduction instead of the 2026 $16,100."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and used an $8,300 standard deduction, a $5,050 personal exemption and a 15% second bracket. The 2026 law (OBBBA) keeps the $16,100 standard deduction, no exemption, and the 10%/12% brackets, which give $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of about $44,000 leaves out the $1,042 401(k) deferral and the $49 IRA deduction, so it should be $42,909. Its $1,541 answer also implies nonrefundable credits this filer does not have. With no credits available, $26,809 of taxable income produces $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $2,080 health premium as a pre-tax deduction without support and assumed the TCJA had expired, applying a standard deduction plus personal exemption. Under 2026 OBBBA law the $16,100 standard deduction with no exemption gives taxable income of $26,809 and tax of $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed the correct $42,909 AGI but used a $15,000 standard deduction and the 2024 $11,600 10% bracket top instead of the 2026 $16,100 and $12,400. Taxable income came out $1,100 too high at $27,909, and the 10% bracket was too narrow."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"With the $1,042 401(k) and $49 IRA deductions and the $16,100 2026 standard deduction, taxable income is $26,809 and tax is $2,969.13. The model's $3,046 is about $77 higher, which matches roughly $640 of extra taxable income from using a pre-2026 standard deduction or 10% bracket edge."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the $2,080 health premium as a pre-tax wage reduction, which pushed AGI down to $40,829 instead of $42,909. It also used a $15,700 standard deduction instead of $16,100, and its bracket arithmetic on its own $25,129 taxable income gives $2,767.48, not the $2,776.98 it reported."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its AGI of $42,909 was correct, but it applied the 'standard deduction and personal exemption under 2026 law' as if the TCJA had expired. The OBBBA made the $16,100 standard deduction, no personal exemption and the 10%/12% brackets permanent, which gives $2,969 instead of $3,788.85."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its AGI of $42,909 was correct, but $3,111 is exactly what 2025 parameters produce: a $15,000 standard deduction and an $11,925 10% bracket. The 2026 $16,100 deduction and $12,400 bracket edge give $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,400 and the 10% bracket top at $12,247. The actual 2026 values are $16,100 and $12,400, so its taxable income was $700 too high at $27,509 and its tax was $3,056.14."
+us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2025 OBBBA single standard deduction of $15,750 instead of the 2026 inflation-adjusted $16,100. That made taxable income $27,159 instead of $26,809 and added $42 of tax at 12%."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It assumed the standard deduction and nonrefundable credits would wipe out the tax. In fact, AGI of $42,909 less the $16,100 standard deduction leaves $26,809 taxable, and this filer qualifies for no nonrefundable credits (no dependents, above the Saver's Credit limit, elderly/disabled credit phased out), so tax is $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed a child support deduction, but child support paid is not deductible. Its $5,199 is also far above what any 10%/12% calculation on the correct $26,809 of taxable income produces ($2,969). The number does not follow from AGI of $42,909 and the $16,100 standard deduction."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"It reached the correct taxable income ($26,809) and brackets (10% on $12,400, 12% on $14,409), but added them up wrong. $1,240 + $1,729.08 = $2,969.08, not the $3,211.08 it reported."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It deducted all $1,351 of retirement contributions, including the Roth 401(k) and Roth IRA amounts, which are after-tax and not deductible. It also subtracted the full $4,510 capital loss instead of the $3,000 cap. Its $3,200 answer does not follow from the $26,809 of taxable income that the correct deductions produce."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 'pre-TCJA rules' for 2026: an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket. The OBBBA kept the TCJA framework, so the $16,100 standard deduction and 10%/12% brackets give $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its $27,159 taxable income implies it used the 2025 $15,750 standard deduction instead of the 2026 $16,100. That overstated taxable income by $350 and tax by $42."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,400 and the 10% bracket top at $12,250 instead of $16,100 and $12,400. As a result it taxed $27,509 instead of $26,809."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA had expired for 2026 and applied the pre-2018 standard deduction plus personal exemption with a 15% bracket. The 2026 law (OBBBA) uses the $16,100 standard deduction with no exemption and the 10%/12% brackets, giving $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized using $13,800 of mortgage interest made up from the $230,000 balance, although no interest paid was listed and unlisted inputs are $0. It also counted the $2,080 premium twice in medical expenses. Without the invented interest, itemized deductions fall below the $16,100 standard deduction, so taxable income is $26,809, not $20,891."
+us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was no answer to score."
+us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It assumed the standard deduction and unnamed credits would bring tax to zero. In fact, $26,809 of taxable income remains after the $16,100 standard deduction, and the filer has no nonrefundable credits, so tax is $2,969."
+us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $960 of workers' compensation to AGI, but workers' compensation is excluded from gross income. It also skipped the $49 IRA deduction and used a $15,750 standard deduction and a $12,150 bracket edge, which gave $3,137.16. It then submitted an unrelated $2,104."
+us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It filed the single, childless head as married filing jointly with MFJ brackets and subtracted a credit for other dependents and child-related credits for dependents who do not exist. It also skipped the 401(k) and IRA deductions and then added an unexplained $2,071.61. A single filer with $26,809 of taxable income and no credits owes $2,969."
us,scenario_015,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no parseable value for federal_refundable_credits, violating the required output contract."
us,scenario_015,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $1,000 refundable Child Tax Credit despite the household having no qualifying child, and it awarded $1,735.80 of EITC even though this childless adult's approximately $43,450 income exceeds the applicable childless EITC range. Both components are $0, so their sum is $0."
us,scenario_015,head_chip_eligible,gemini-3.5-flash-lite,llm_error,other,False,"The model applied the governing rule correctly — CHIP's categorical scope is limited to children under 19 (plus pregnant women at state option), so a 36-year-old head fails the age gate before any income screen — and its explanation ends ""Head is 36 years old, so CHIP eligibility is false. value = 0"". Its submitted numeric value for head_chip_eligible was nevertheless 1, contradicting its own explanation and violating the prompt's rule that a 1 be submitted only when the explanation says eligible. The defect is in the emitted binary, not in the derivation: the stated reasoning matches the reference's chip_category = 3 / is_chip_eligible = False."
@@ -1217,6 +1235,53 @@ us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,F
us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model decided that being disabled qualifies an adult under 65 for Medicare. That skips the requirement of 24 months of Social Security Disability Insurance entitlement. The 36-year-old head has no SSDI income and no ESRD or ALS status, so they are not Medicare eligible."
us,scenario_015,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model based eligibility only on the head being listed as disabled. Disability-based Medicare requires 24 months of SSDI benefit receipt, and a disability flag alone doesn't meet that. The head is 36 and gets no Social Security disability benefits, so the answer is not eligible (0)."
us,scenario_015,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the head's disabled status as enough for Medicare. Under-65 eligibility requires 24 months of SSDI entitlement, or ESRD or ALS status. The head is 36 and has none of these, so Medicare eligibility is 0."
+us,scenario_015,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model treated the four-city list (NYC/Philadelphia/KC/St. Louis) as the full scope. The other reading counts every local income tax PolicyEngine models, including Indiana county tax. Under that reading, the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana county taxes out of the output because they were not in the four-city list. PolicyEngine's local_income_tax includes Indiana county income tax: Adams County's 1.6% on $41,909.38 of Indiana AGI is $670.55. Excluding it under the enumerated-list reading produced $0."
+us,scenario_015,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model wrongly said Indiana has no local income tax. All 92 Indiana counties levy a county income tax on Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38 ($42,909.38 federal AGI minus the $1,000 exemption), the tax is $670.55."
+us,scenario_015,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model decided Indiana county income tax falls outside the output because only four cities were named. PolicyEngine counts the county tax in local_income_tax, applying Adams County's 1.6% to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model read the four named cities as the only taxes in this output and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax. The default Adams County rate of 1.6% on $41,909.38 of Indiana AGI gives $670.55."
+us,scenario_015,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model took the output to cover only NYC, Philadelphia, KC and St. Louis taxes, so it skipped Indiana's county income tax. PolicyEngine includes that tax: 1.6% (Adams County) on Indiana AGI of $41,909.38 is $670.55."
+us,scenario_015,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model left Indiana county income tax out because of the four-city list. PolicyEngine's local_income_tax includes it and, with no county given, uses Adams County's 1.6% rate on Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the named jurisdictions and found none that applied. It never added Indiana county income tax, which PolicyEngine includes in local_income_tax: Adams County's 1.6% times $41,909.38 of Indiana AGI is $670.55."
+us,scenario_015,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and concluded nothing applies to an Indiana resident. It missed Indiana county income tax, which PolicyEngine counts in local_income_tax at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55."
+us,scenario_015,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model treated the four named city and earnings taxes as the full scope and ignored Indiana's county income tax. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $41,909.38 ($42,909.38 federal AGI minus the $1,000 exemption), giving $670.55."
+us,scenario_015,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model read the four cities as the only jurisdictions covered and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% (Adams County) on $41,909.38 is $670.55."
+us,scenario_015,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in one of the four named cities. It skipped Indiana county income tax, which PolicyEngine includes at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55."
+us,scenario_015,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model applied only the four-city list and left out Indiana's county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and missed Indiana county income tax. PolicyEngine includes that tax in local_income_tax: Adams County's 1.6% on Indiana AGI of $41,909.38 is $670.55."
+us,scenario_015,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies in Indiana, but every Indiana county levies an income tax on Indiana AGI. The default Adams County rate of 1.6% on $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model recognized that Indiana has county income taxes but excluded them because of the four-city list. PolicyEngine counts them in local_income_tax, applying Adams County's 1.6% to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies and never considered Indiana's county income tax. PolicyEngine levies it at Adams County's 1.6% on Indiana AGI of $41,909.38 (federal AGI of $42,909.38 minus the $1,000 exemption), giving $670.55."
+us,scenario_015,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four named city taxes and excluded Indiana county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It missed Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55."
+us,scenario_015,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model said Indiana has no local income tax, but Indiana county income tax applies statewide to Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38, the tax is $670.55."
+us,scenario_015,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model treated the four named cities as the only local taxes and left out Indiana county income tax. PolicyEngine includes it in local_income_tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model assumed PolicyEngine models only NYC, Philadelphia, KC and St. Louis local taxes. PolicyEngine also models Indiana county income tax and applies Adams County's 1.6% to Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model read the four-city list as the full output set and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% (Adams County default) on Indiana AGI of $41,909.38 is $670.55."
+us,scenario_015,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because Indiana is not among the four listed jurisdictions. It skipped Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model applied only the four listed city taxes and left out Indiana's county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model found no listed jurisdiction and answered $0, never considering Indiana county income tax. PolicyEngine counts that tax in local_income_tax: Adams County's 1.6% times $41,909.38 of Indiana AGI is $670.55."
+us,scenario_015,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model treated the missing county as meaning no local tax. PolicyEngine instead assigns a default county (Adams) and applies its 1.6% county income tax rate to Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, KC and St. Louis taxes and excluded Indiana county income tax. PolicyEngine includes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model read the listed jurisdictions as exhaustive and, with no county given, answered $0. PolicyEngine defaults to Adams County and applies its 1.6% county income tax to Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model assumed only the four separately modeled city taxes feed this output. PolicyEngine's local_income_tax also includes Indiana county income tax, computed at Adams County's 1.6% on $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model found Indiana absent from the listed local-tax jurisdictions and answered $0. PolicyEngine levies Indiana county income tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model limited the output to the named cities and left out Indiana county income tax. PolicyEngine includes that tax in local_income_tax at Adams County's 1.6% of $41,909.38 Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and answered $0. PolicyEngine also computes Indiana county income tax: 1.6% (Adams County) on Indiana AGI of $41,909.38 is $670.55."
+us,scenario_015,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is outside the four named cities. It missed Indiana county income tax, which PolicyEngine includes at Adams County's 1.6% on $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model read the output as covering only the listed cities and left out Indiana's county income tax. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies in Indiana, overlooking the county income tax every Indiana county levies on Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38, the tax is $670.55."
+us,scenario_015,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model applied only the four listed city taxes and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, which gives $670.55."
+us,scenario_015,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model treated NYC, Philadelphia, KC and St. Louis as the only covered taxes and excluded Indiana county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in one of the four named cities. It skipped Indiana county income tax, which PolicyEngine levies at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55."
+us,scenario_015,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four specified categories and left out Indiana county income tax. PolicyEngine includes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model read the four-city list as exhaustive and answered $0. PolicyEngine counts Indiana county income tax in local_income_tax, applying the default Adams County rate of 1.6% to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city and earnings taxes and concluded none apply. It missed Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% on $41,909.38 of Indiana AGI ($42,909.38 federal AGI minus the $1,000 exemption), giving $670.55."
+us,scenario_015,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model explicitly placed Indiana county taxes outside this output's scope. PolicyEngine includes them in local_income_tax at Adams County's 1.6% on Indiana AGI of $41,909.38, giving $670.55."
+us,scenario_015,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model assumed PolicyEngine's local-income-tax output covers only NYC, Philadelphia, KC and St. Louis. PolicyEngine also includes Indiana county income tax and applies Adams County's 1.6% to $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model excluded Indiana county income tax from the output and took the missing county as meaning zero tax. PolicyEngine includes the county tax and defaults to Adams County, applying 1.6% to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed jurisdictions and answered $0, leaving out Indiana county income tax. PolicyEngine computes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55."
+us,scenario_015,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model checked only the four named city taxes and missed Indiana county income tax. PolicyEngine includes that tax in local_income_tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55."
us,scenario_015,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted the $2,080 health insurance premiums from wages as a pre-tax cafeteria-plan exclusion and applied 7.65% to $42,920. The FICA base here is the full $45,000 of gross wages, since the listed premiums are not a Section 125 salary reduction. The correct total is $2,790 + $652.50 = $3,442.50."
us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It lowered FICA wages to $42,920 by treating the $2,080 health insurance premiums as a pre-tax payroll deduction. Employee Social Security and Medicare tax apply to the full $45,000 of gross wages, which gives $3,442.50 rather than $3,283."
us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $2,080 employer-sponsored insurance premiums were pre-tax and cut FICA wages to $42,920. The 6.2% and 1.45% employee rates apply to the full $45,000 of gross wages, giving $2,790 + $652.50 = $3,442.50."
@@ -1226,38 +1291,39 @@ us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It found the cor
us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value and no explanation for payroll_tax. The correct derivation is 7.65% of $45,000 in wages: $2,790 Social Security plus $652.50 Medicare, or $3,442.50."
us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It correctly reached $3,442.50 from $2,790 Social Security and $652.50 Medicare, then raised it to $3,495 for made-up 'Indiana county tax adjustments'. Indiana county taxes are local income taxes, not employee payroll taxes, and Indiana has no mandatory employee state payroll tax. Nothing should be added to the FICA total."
us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It subtracted the $1,042 traditional 401(k) deferral from the FICA base. Under IRC 3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base stays at the full $45,000. It then submitted $3,425.67, which does not match its own computed total of $3,362.79. The correct amount is $3,442.50."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly built Indiana taxable income of $41,909 (federal AGI $42,909 less the $1,000 exemption) and applied the 2026 2.95% rate to get about $1,236. It then cut the answer to $1,234 for an invented 'rounding in exemption computation' that has no basis, since the exemption is a flat $1,000."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,510 capital loss instead of the $3,000 cap, wrongly included $960 of nontaxable workers' compensation, and skipped the $1,000 Indiana exemption. It used a 3.15% rate instead of the 2026 rate of 2.95%, then made an arbitrary cut for 'disability exemptions' to reach $1,255 instead of $41,909 x 2.95% = $1,236."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It invented a $500 exemption for a disabled filer under 65 (Indiana's additional exemptions are for age 65+ and blindness only) and used a 3.05% rate instead of the 2026 rate of 2.95%. It then submitted $1,352, which does not match its own $1,263 result."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It failed to subtract the $1,042 traditional 401(k) deferral from wages, which put federal AGI at $43,951 instead of $42,909. It then pushed its own 2.95% result ($1,267) up to $1,392 by switching to a 3.05% rate, and even at 3.05% its base would not produce that figure."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a 2.85% rate instead of the 2026 rate of 2.95%. It also put federal AGI at $42,441, about $468 too low, instead of $42,909. Applying 2.95% to $41,909 gives $1,236."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but cut Indiana taxable income to $38,829 with two deductions Indiana does not allow. One was a $1,000 'disability exemption' (Indiana's additional exemptions cover only age 65+ and blindness). The other was a $2,080 deduction for health insurance premiums. Only the $1,000 personal exemption applies, which leaves $41,909."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 3.05% rate instead of the 2026 rate of 2.95% and subtracted an extra $1,000 exemption for being disabled, which Indiana does not give a disabled filer under 65. It then inflated its own $1,248 to $1,350 for 'county/local considerations', even though the requested output excludes local tax."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,age_disability,False,"It used the correct 2.95% rate and $42,909 AGI but subtracted $2,000 of exemptions by adding a $1,000 exemption for being disabled. Indiana's additional $1,000 exemptions apply only to filers aged 65+ or blind, so taxable income is $41,909, not $40,909."
-us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $40,829, which subtracts the $2,080 of health insurance premiums; those are not an above-the-line deduction. It also applied 3.05% instead of the 2026 rate of 2.95%. The correct figure is $41,909 x 2.95% = $1,236."
-us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used an AGI of $40,829, reduced by the $2,080 of premiums that are not deductible. It then invented a $1,000 Indiana standard deduction and a $1,000 disabled exemption, neither of which exists, so it taxed $37,829 instead of $41,909 at 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a $3,000 Indiana deduction for child support paid and a $1,500 exemption for being disabled; Indiana allows neither. It also used a 3.0% rate instead of the 2026 rate of 2.95%, taxing $37,409 instead of $41,909."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied a made-up 2.79% rate. Its $1,256 implies a base of about $45,000, meaning it taxed roughly gross wages without the AGI adjustments (401(k), IRA, capped capital loss) or the $1,000 exemption. The correct figure is $41,909 x 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate and $1,000 exemption but started from a federal AGI of $40,829, which wrongly subtracts the $2,080 of health insurance premiums. The correct AGI is $42,909."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It built the correct Indiana taxable income of $41,909 but applied the 2024 rate of 3.05% instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $1,373 equals 3.05% of the $45,000 gross wages. It skipped the AGI adjustments (401(k), IRA, capped capital loss, interest) and the $1,000 exemption, used the 2024 rate instead of 2.95%, and wrongly mixed county tax into a state-only output."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $40,829, which wrongly subtracts the $2,080 of health insurance premiums, and it applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%. It also built federal AGI as $42,490 by deducting the full $4,510 capital loss instead of the $3,000 cap and by omitting the $1,042 traditional 401(k) and $49 IRA deductions."
-us,scenario_015,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied 2.90%, the rate scheduled for 2027, instead of the 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It applied a flat rate after 'standard deductions and exemptions', but Indiana has no standard deduction. Its $1,018 implies a base of about $34,500, far below the correct $41,909 (federal AGI less only the $1,000 exemption), taxed at 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits wipe out Indiana tax. In fact, with $42,909 of AGI, only a $1,000 exemption and no nonrefundable credits available, the 2.95% flat tax leaves $1,236 owed."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a made-up 3.025% rate instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but subtracted an invented extra $1,000 'Indiana deduction' on top of the $1,000 personal exemption, which put taxable income at $40,909 instead of $41,909."
-us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied a flat rate to roughly the $45,000 of wages after an invented 'standard deduction'. Its $1,400 implies a base above $45,000 at 3.05%, so it skipped the federal AGI adjustments and the $1,000 exemption, and it did not use 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a 2.85% rate instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied the 2024 rate of 3.05% instead of the 2026 rate of 2.95%. Its $1,230 implies a base of about $40,300, which means it subtracted more than the single $1,000 exemption from the $42,909 AGI."
-us,scenario_015,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a made-up 2.97% rate instead of the scheduled 2026 rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrongly included $960 of nontaxable workers' compensation, deducted the full $4,510 capital loss instead of the $3,000 cap, and skipped the $1,000 Indiana personal exemption. It then applied the 2024 rate of 3.05% instead of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the rate to federal taxable income ($30,860, after the federal standard deduction) rather than to Indiana AGI less the $1,000 exemption ($41,909). It also used 3.05% instead of 2.95% and rounded its $941 up to $950."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model got the correct base ($41,909) and rate (2.95%) and arrived at about $1,236. It then cut the answer to $1,234, citing 'minor rounding in exemption computation'. No such rounding exists, because Indiana's exemption is exactly $1,000."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model's AGI was wrong in two ways: it included nontaxable workers' compensation, and it deducted the full $4,510 capital loss instead of the $3,000 cap. It never subtracted the $1,000 Indiana exemption and used an outdated 3.15% rate. It then applied an unexplained 'disability' adjustment to reach $1,255."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model used a 3.05% rate instead of Indiana's 2026 rate of 2.95%. It also never computed a figure, submitting a round $1,000 instead of applying the rate to the $41,909 base, which gives $1,236."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model added a $500 disabled exemption that Indiana does not grant to a non-blind taxpayer under 65, and used 3.05% instead of the 2026 rate of 2.95%. Its submitted $1,352 does not even match its own $1,263 calculation."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model did not exclude the $1,042 traditional 401(k) deferral from wages, which put AGI at $43,951 instead of $42,909. It then dropped its correct 2.95% calculation and inflated the answer to $1,392 using a 3.05% 'adjustment'."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used 2.85% instead of Indiana's 2026 rate of 2.95%. Its AGI of $42,441 also deducts the full $4,510 capital loss instead of the $3,000 cap and skips the $1,042 401(k) exclusion."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model's rate (2.95%) and AGI ($42,909) were right, but it subtracted two deductions that do not apply: a $1,000 'disability exemption', which Indiana does not grant a non-blind taxpayer under 65, and a $2,080 health insurance premium deduction, which Indiana does not offer. That cut taxable income to $38,829 instead of $41,909."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used 3.05% instead of 2.95% and added a $1,000 disability exemption that does not apply to this taxpayer. It then raised its own $1,248 result to $1,350 with no stated basis."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,age_disability,False,"The rate (2.95%) and AGI were right, but the model subtracted an extra $1,000 'disability' exemption. Indiana's additional $1,000 exemptions apply only to taxpayers who are 65 or older or blind, so only the $1,000 personal exemption applies."
+us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $40,829, which means it wrongly subtracted the $2,080 health insurance premiums from the $42,909 federal AGI. It also used 3.05% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $2,080 premiums from AGI and invented a $1,000 'standard' deduction (Indiana has none). It also claimed a $1,000 disabled exemption that does not apply to a non-blind taxpayer under 65, leaving $37,829 instead of $41,909."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $3,000 child support deduction (Indiana has none) and a $1,500 disabled exemption for the taxpayer. It also used a 3.0% rate instead of 2.95%. The correct taxable base is $42,909 minus only the $1,000 personal exemption."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 2.79% rate instead of 2.95%. Its $1,256 implies a base of about $45,000, so it taxed gross wages without the 401(k), IRA, capital-loss, and exemption adjustments that bring the base to $41,909."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model began from an AGI of $40,829, which means it wrongly subtracted the $2,080 health insurance premiums. Applied to the correct $42,909 AGI minus the $1,000 exemption, its 2.95% rate would have given $1,236."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 3.05% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used 3.05% instead of 2.95% and wrongly said it was adding county taxes, which belong in local tax, not state tax. Its $1,373 implies a base of about $45,000, so it skipped the AGI adjustments and the $1,000 exemption."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $40,829, which means it wrongly subtracted the $2,080 health premiums from the $42,909 federal AGI. It also used 3.0% instead of the 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 3.0% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 3.0% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 3.0% instead of 2.95%. Its AGI of $42,490 also deducts the full $4,510 capital loss instead of the $3,000 cap and ignores the $1,042 401(k) exclusion and the $49 IRA deduction."
+us,scenario_015,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 2.90% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model applied 'standard deductions', but Indiana has no standard deduction; it allows only the $1,000 personal exemption from the $42,909 federal AGI. Its $1,018 is consistent with taxing about $34,500, which understates the $41,909 base."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model reported zero by assuming that deductions and credits wipe out the tax. No nonrefundable Indiana credit applies, so 2.95% of $41,909 gives $1,236 owed."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 3.025% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted an invented $1,000 'Indiana deduction' on top of the $1,000 personal exemption, cutting the base to $40,909. Indiana allows only the single $1,000 exemption for this filer, which leaves $41,909."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a 'standard deduction' that Indiana does not have and worked from $45,000 of wages instead of the $42,909 federal AGI. Its $1,400 implies a base above AGI at 2.95%, so it overstated either the base or the rate."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 3% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 2.85% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or reasoning for state_income_tax_before_refundable_credits, so its answer is missing."
+us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used 3.05% instead of 2.95%. Its $1,230 implies a base of about $40,300 instead of the correct $41,909, which is $42,909 federal AGI minus the $1,000 exemption."
+us,scenario_015,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model's $41,909 base was correct, but it used 2.97% instead of Indiana's 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used 3.05% instead of 2.95%. It also omitted the $1,000 personal exemption, counted nontaxable workers' compensation in AGI, and deducted the full $4,510 capital loss instead of the $3,000 cap."
+us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the tax to federal taxable income of $30,860, which is after the federal standard deduction. Indiana's base is federal AGI ($42,909) minus the $1,000 exemption. It also used 3.05% instead of 2.95%."
us,scenario_015,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_016,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly derived that itemizing the medical expenses against the 7.5%-of-AGI floor ($2,410) drives taxable income to zero and stated ""tax = 0,"" then discarded that result on an invented benchmark convention that a wage earner's self-paid premiums are pre-tax and therefore not itemizable. Having reverted to the standard deduction it computed $1,677 ($12,400 at 10% plus $3,638 at 12%), yet submitted $1,523, a figure that matches neither the itemized nor the standard-deduction branch of its own work."
us,scenario_016,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never compared itemized deductions against the standard deduction, missing the roughly $44,590 net medical expense deduction ($47,000 of premiums, other medical, and OTC expenses less the $2,410 7.5%-of-AGI floor) that exceeds AGI and zeroes taxable income. It compounded this by deducting the full $3,000 of student loan interest instead of the $2,500 statutory cap, using a $15,000 standard deduction, fabricating $600 of mortgage interest from a $20,000 balance, and awarding a $575 Saver's Credit even though no retirement contributions are listed and unlisted inputs are zero."
@@ -1288,43 +1354,44 @@ us,scenario_016,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wron
us,scenario_016,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_016,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_016,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model got AGI ($61,592), the $16,100 standard deduction and the 2026 brackets right, then also subtracted a $1,625 OBBBA car-loan interest deduction. That deduction requires a qualifying new, US-assembled vehicle, and the household lists no such vehicle. The extra deduction cut taxable income to $43,867 instead of $45,492 and the tax to $5,016 instead of $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Beyond the correct $16,100 standard deduction, the model subtracted a $1,625 car-loan interest deduction and a $6,760 overtime premium deduction that it inferred from 50 weekly hours. No qualified overtime compensation is listed, so the overtime deduction is $0. Wages of $68,005 are close to straight-time pay at $26 × 50 × 52, so no time-and-a-half premium is embedded. Neither deduction applies, so taxable income is $45,492, not $37,107."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It also used a bogus bracket structure: 12% on everything up to $46,225 with no 10% bracket, then 22% above that, when the 2026 12% bracket runs to $50,400. It computed $5,716, then submitted an unrelated $4,903."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed the correct liability, $1,240 + 12% × $33,092 = $5,211 on taxable income of $45,492. It then threw that result away for an unexplained 'adjusting with rounded brackets' figure of $6,754, which no bracket computation on $45,492 produces."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model set up the correct computation, $1,240 on the first $12,400 plus $3,971 at 12% on $33,092, which totals $5,211. It then misadded the two pieces and reported $6,033."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed FLSA non-exempt status and took a $6,760 overtime premium deduction (0.5 × $26 × 10 hours × 52 weeks). It also assumed the loan qualified and took a $1,625 car-loan interest deduction. Unlisted status facts are false and unlisted amounts are zero, so neither deduction applies. The standard deduction alone gives taxable income of $45,492, not $37,107."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model subtracted a $200 saver's credit at a 10% rate using a $64,000 AGI ceiling. The single-filer ceiling is about $40,000, so AGI of $61,592 gets no credit. It also used the 2025 pre-OBBBA $15,000 standard deduction and $11,925 bracket top instead of 2026's $16,100 and $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model reached the correct taxable income of $45,492 but used a $12,150 top for the 10% bracket instead of $12,400, giving $5,216. It then cut about $303 more to reach $4,913, even though its own reasoning says the saver's credit phases out at this income and no other nonrefundable credit applies."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model counted $18,000 of health premiums by adding both $9,000 premium lines, but they describe the same $9,000 premium. That inflated medical expenses to $20,000 and itemized deductions to $16,535, so the model itemized. With the premium counted once, itemized deductions fall far below the $16,100 standard deduction. It also used a $12,075 top for the 10% bracket instead of $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026, applying a $7,700 standard deduction, a $4,950 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, with a $16,100 standard deduction and 10% and 12% brackets. It also subtracted the $9,000 health premium from AGI as a pre-tax payroll deduction, which the facts do not list, giving AGI of $52,592 instead of $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model got AGI right at $61,592 but used a $15,650 standard deduction and a $12,450 top for the 10% bracket. The 2026 values are $16,100 and $12,400. As a result it overstated taxable income by $450, at $45,942 instead of $45,492."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and the 15% and 25% brackets. OBBBA made the TCJA brackets permanent and set the 2026 single standard deduction at $16,100, with no personal exemption. Its taxable income of $47,992 and 25%-bracket tax are both wrong as a result."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model left out the $289 IRA deduction and the $50 of interest, and used the 2024 standard deduction of $14,600 to reach taxable income of $47,231. It then applied a flat 10% ($4,724) to all of it instead of taxing the amount above $12,400 at 12%."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used projected pre-TCJA 2026 law, with a combined standard deduction and personal exemption of $13,790 and pre-TCJA 15% and 25% brackets. The OBBBA-extended structure applies instead: a $16,100 standard deduction, no exemption, and 10% and 12% brackets on $45,492 of taxable income."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied TCJA-sunset law, with an $8,300 standard deduction, a $5,300 exemption and a 15% bracket starting at $11,000, instead of the OBBBA 2026 $16,100 standard deduction and 10% and 12% brackets. It also subtracted the $9,000 health premium from AGI as a pre-tax deduction, which the facts do not list, giving AGI of $52,592 instead of $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no figures. Its $5,304 is about $93 above the correct $5,211.05, which fits pre-2026 parameters: a standard deduction around $15,000 to $15,400 and an older, lower 10% bracket top. The correct 2026 values are a $16,100 standard deduction and a $12,400 bracket top, applied to AGI of $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got AGI right at $61,592 but used a $15,700 standard deduction and a $12,275 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $45,892 instead of $45,492."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed TCJA provisions expired in 2026 and subtracted a pre-TCJA standard deduction plus a personal exemption, reaching taxable income of $48,142 taxed at 10% and 15%. OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no exemption, and 10% and 12% brackets."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied 2025 pre-OBBBA parameters: a $15,000 standard deduction and a $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400. That gave taxable income of $46,592 and tax of $5,352.54."
-us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model estimated the 2026 standard deduction at $15,400 and the 10% bracket top at $12,260. The actual values are $16,100 and $12,400. That overstated taxable income by $700, at $46,192 instead of $45,492."
-us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of the 2026 OBBBA amount of $16,100, giving taxable income of $46,592. It also used about $12,300 for the 10% bracket top instead of $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model subtracted a nonrefundable child tax credit for 'one qualifying child', but the household is a single adult with no dependents, so no CTC is available. The tax before refundable credits is the full $5,211.05 on $45,492 of taxable income."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model asserted zero liability without computing any tax. $68,005 of wages leaves $45,492 of taxable income after the 401(k), IRA and $16,100 standard deduction, which produces $5,211.05 of tax at the 10% and 12% rates, with no credits to offset it."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction, inferred from 50 weekly hours at $26. No qualified overtime compensation is listed, and wages of $68,005 are close to straight-time pay for all hours, so the deduction is $0. Taxable income is $45,492, not $38,732."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model got AGI and the $16,100 standard deduction right, but its $5,220.54 implies a $11,925 top for the 10% bracket, the 2025 pre-OBBBA figure. The 2026 top is $12,400, which gives $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model took an estimated $6,760 qualified overtime premium deduction (10 hours × $13 × 52). No qualified overtime compensation is listed, so the deduction is $0, and wages of $68,005 are close to straight-time pay at $26 × 50 × 52. Taxable income is $45,492, not $38,732."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction that none of the listed inputs support. Unlisted amounts are $0, and the stated wages already reflect straight-time pay for 50 hours. That lowered taxable income to $38,732 instead of $45,492."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model gave a rounded guess of about $5,200 without doing the bracket computation. Exact 2026 parameters (a $16,100 standard deduction, a $12,400 10% bracket top and 12% above it) on $61,592 of AGI give $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied TCJA-sunset law: a $5,300 personal exemption and pre-TCJA 10% and 15% brackets. It also itemized $15,035 using $18,500 of medical expenses, which counts the $9,000 premium twice. Under OBBBA's permanent structure the $16,100 standard deduction exceeds the correctly counted itemized total, and no exemption exists."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and a $12,250 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $46,192 instead of $45,492."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got the $16,100 standard deduction and $45,492 of taxable income right, but capped the 10% bracket at $12,200 instead of the 2026 top of $12,400. That overstated tax by $4 ($5,215.04 instead of $5,211.05)."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 standard deduction instead of the 2026 OBBBA amount of $16,100, giving taxable income of $46,217 instead of $45,492. The tax is overstated as a result."
-us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model itemized about $15,381 of medical deductions, which assumes $20,000 of medical costs, adding both $9,000 premium lines as separate premiums. With the $9,000 premium counted once, medical deductions plus state tax fall well below the $16,100 standard deduction, which leaves taxable income at $45,492, not $45,132."
-us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so its answer is missing, not a substantive miscalculation."
-us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model gave $0 with no reasoning, which implies it treated the filer as having no tax liability. AGI of $61,592 less the $16,100 standard deduction leaves $45,492 of taxable income, which produces $5,211.05 of tax, and no nonrefundable credits apply."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the 2025 OBBBA standard deduction of $15,750 and a $12,550 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. It also left out the deductible $289 traditional IRA contribution, putting AGI at $61,881 instead of $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted extra 'qualifying retirement contributions' to reach taxable income of $32,942, $12,550 below the correct $45,492. Only the traditional 401(k) and $289 IRA reduce AGI, and the Roth contributions do not. It also subtracted a $303 saver's credit, but AGI of $61,592 is well above the single-filer eligibility ceiling."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its AGI, $16,100 standard deduction and brackets were all correct. It then also subtracted the $1,625 car-loan interest under the OBBBA deduction, but that deduction requires interest on a loan for a new, U.S.-assembled personal vehicle, which the facts don't establish. That understated taxable income by $1,625 and tax by $195."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime-premium deduction it built from 50 weekly hours and a $1,625 car-loan interest deduction. The facts establish neither qualified overtime compensation nor a qualifying new-vehicle loan, so taxable income is $45,492, not $37,107."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It taxed the first $46,225 entirely at 12%, skipping the 10% bracket, and invented a 22% bracket. It then submitted $4,903, which doesn't match its own $5,716 calculation."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the health premiums from wages, but they are not listed as pre-tax payroll deductions and don't reduce AGI. It then gave a rough $4,700 instead of taxing $45,492 at 10% up to $12,400 and 12% above. Its own stated $50,000 taxable income would produce about $5,752, not $4,700."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed $1,240 + 12% × $33,092 = $5,211. It then discarded that result for an unexplained 'rounded bracket' figure of $6,754, so the final answer contradicts its own correct calculation."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It set up the correct calculation, $1,240 + $3,971 on $45,492 of taxable income, which totals $5,211. It then reported $6,033, an addition error with no basis in the brackets it cited."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It assumed FLSA non-exempt status to claim a $6,760 overtime-premium deduction and assumed the car loan qualifies for a $1,625 interest deduction. The facts support neither, so taxable income is $45,492, not $37,107."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% ceiling of $11,925 instead of 2026's $16,100 and $12,400. It also claimed a $200 Saver's Credit using wrong income limits; a single filer's AGI of $61,592 is far above the single ceiling of about $40k, so the credit is $0."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 10% bracket ceiling of $12,150 instead of $12,400, which gave $5,216. It then subtracted an unexplained ~$303 to reach $4,913, even though it acknowledged the Saver's Credit is phased out at this income and no other nonrefundable credit applies."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $9,000 premium twice as $18,000 and treated $1,500 of over-the-counter costs as deductible medical expenses. That inflated itemized deductions to $16,535, which it compared against a wrong $15,400 standard deduction. The correct medical-plus-SALT itemized total is well below the $16,100 standard deduction. It also used a wrong 10% ceiling of $12,075."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,700 standard deduction, a $4,950 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, giving a $16,100 standard deduction, no exemption and 10%/12% brackets. It also improperly subtracted $9,000 of health premiums from wages."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and a $12,450 10% ceiling instead of the 2026 values of $16,100 and $12,400. That overstated taxable income by $450 and tax by about $53."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA kept the TCJA structure for 2026: a $16,100 standard deduction, no exemption, and 10% to $12,400 then 12%."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2024 $14,600 standard deduction instead of $16,100 and left out the $289 IRA deduction and the $50 of interest. Its $4,724 doesn't follow from any bracket calculation on its own $47,231 of taxable income."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used pre-TCJA 'sunset' law, with a combined deduction and personal exemption of $13,790 and 15%/25% brackets. The 2026 law has a $16,100 standard deduction, no personal exemption, and 10%/12% brackets up to $50,400."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, applying an $8,300 standard deduction, a $5,300 exemption and a 15% bracket instead of 2026's $16,100 deduction and 10%/12% brackets. It also subtracted $9,000 of health premiums from wages even though they aren't a listed pre-tax deduction."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation. Its $5,304 is about $93 above the correct tax, which matches a standard deduction roughly $775 smaller than 2026's $16,100, meaning it used outdated deduction or bracket parameters."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction and a $12,275 10% ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income by $400."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a standard deduction plus personal exemption with 10%/15% brackets. OBBBA continues the $16,100 standard deduction, with no exemption and 10%/12% rates, for 2026."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% ceiling of $11,925 instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $46,592 instead of $45,492."
+us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,400 and the 10% ceiling at $12,260 instead of the published $16,100 and $12,400. That overstated taxable income by $700."
+us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction (the 2025 pre-OBBBA figure) instead of 2026's $16,100, and a $12,300 10% ceiling instead of $12,400. Taxable income came out as $46,592 instead of $45,492."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It subtracted a nonrefundable Child Tax Credit for 'one qualifying child,' but the household is a single adult with no dependents. No credit applies against the $5,211.05 of regular tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed there was no remaining liability. A single filer with $61,592 AGI has $45,492 of taxable income after the $16,100 standard deduction and owes $5,211.05, with no nonrefundable credits to offset it."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 qualified-overtime deduction calculated from the 50-hour work week. The facts establish no qualified overtime compensation, so taxable income is $45,492, not $38,732."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its taxable income of $45,492 is correct, but its $5,220.54 implies it used the 2025 10% ceiling of $11,925 instead of 2026's $12,400. That overstated tax by $9.50."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It added an estimated $6,760 overtime-premium deduction (10 hours × $13 × 52 weeks) on top of the $16,100 standard deduction. The facts establish no qualified overtime compensation, so taxable income is $45,492."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It deducted $6,760 of qualified overtime, reducing taxable income to $38,732. No overtime deduction applies here, so the tax on $45,492 is $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It did no bracket calculation, mislabeled roughly $61k of AGI as taxable income, and gave a round $5,200. The correct calculation is $1,240 + 12% × $33,092 = $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a repealed $5,300 personal exemption and 10%/15% sunset brackets. It also built a $15,035 itemized total by counting $18,000 of premiums. The 2026 law gives a $16,100 standard deduction with no exemption, so taxable income is $45,492."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,250 10% ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income by $700."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its taxable income of $45,492 is correct, but it ended the 10% bracket at $12,200 instead of 2026's $12,400. That pushed $200 more into the 12% bracket and gave $5,215.04."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $15,375 standard deduction instead of the 2026 single amount of $16,100. That overstated taxable income by $725, to $46,217."
+us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It claimed about $15,381 of medical deductions by counting the $9,000 premium twice and including over-the-counter costs. That made itemized deductions of about $16,461 appear larger than the $16,100 standard deduction. The correct itemized total is far below $16,100, so the standard deduction applies and taxable income is $45,492."
+us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this variable, so there is no usable answer."
+us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It gave $0 with no derivation. A single filer with $61,592 AGI and $45,492 of taxable income after the $16,100 standard deduction owes $5,211.05, with no nonrefundable credits to reduce it."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It left out the $289 deductible traditional IRA contribution, giving AGI of $61,881. It also used the $15,750 standard deduction (the 2025 figure) instead of 2026's $16,100, and a $12,550 10% ceiling instead of $12,400."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It understated taxable income by about $12,550 ($32,942 instead of $45,492) by subtracting retirement contributions beyond the 401(k) and IRA amounts already removed from AGI. It then claimed a $303 Saver's Credit even though a single filer's AGI of $61,592 is far above the credit's income ceiling."
us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies."
us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38."
@@ -1334,136 +1401,128 @@ us,scenario_018,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated
us,scenario_018,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract."
us,scenario_018,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits federal employee FICA on $68,005 of wages. The correct computation includes $4,216.31 of Social Security tax and $986.07 of Medicare tax, totaling $5,202.38."
us,scenario_018,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning correctly reached $61,592 − $15,750 = $45,842 × 2.5% = $1,146. It then subtracted the $1,625 auto loan interest as a federal car-loan deduction, which Arizona taxable income does not include here. It finally submitted $1,503, a number that none of its own arithmetic supports."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the federal 2026 single standard deduction of $16,100 instead of Arizona's $15,750. That understated taxable income ($45,492 vs $45,842.09) and gave $1,137.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied repealed graduated brackets (2.55%/3.34%) instead of Arizona's flat 2.5% rate. It also used a $14,600 standard deduction instead of $15,750 and misadded AGI as $61,567. It then submitted $2,175, which does not match even its own $1,304 bracket computation."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It guessed an Arizona standard deduction of $15,000 instead of $15,750. That gave taxable income of $46,592 and tax of $1,165 instead of $45,842.09 and $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a $14,600 standard deduction instead of $15,750 and computed $1,175. It then inflated that to $1,471 through an unexplained 'indexing' adjustment that moves in the wrong direction for a larger deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of Arizona's $15,750 and reached $1,137. It then submitted $1,481, which contradicts its own stated final computation."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the federal 2026 standard deduction of $16,100 instead of Arizona's $15,750. Taxable income came out $350 too low ($45,492) and tax was $1,137.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It projected the Arizona single standard deduction at $14,600 instead of $15,750. That overstated taxable income at $46,992 and gave $1,175 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $14,600 standard deduction instead of $15,750 and reached $1,175. It then added an invented 'HSA-like premium' adjustment to submit $1,450, although no such addition to Arizona taxable income exists."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the federal $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30 instead of $45,842.09 and $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $15,400 Arizona standard deduction instead of $15,750. Taxable income came out at $46,192 and tax at $1,154.80 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut AGI to $52,592 by excluding the $9,000 health premiums as pre-tax, even though AGI is $61,592.09. It then used a $7,700 standard deduction instead of $15,750. The two errors partly offset and gave $1,122.30."
-us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $15,650 Arizona standard deduction instead of $15,750. That overstated taxable income by $100 ($45,942) and gave $1,148.55 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of $15,750 and also subtracted a $2,100 personal exemption. Arizona has no taxpayer personal exemption, so its taxable income of $43,392 is too low and gave $1,084.80."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It projected a $14,600 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,992 and tax of $1,174.80 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. The correct figure is 2.5% × ($61,592.09 − $15,750) = $1,146.05, while its $1,400 matches 2.5% of about $56,000 of taxable income. That means it applied far less than the $15,750 standard deduction, or simply rounded a guess."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a stale $13,790 standard deduction instead of Arizona's 2026 $15,750. That overstated taxable income at $47,802 and gave $1,195."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $52,592, excluding $9,000 of health premiums that are not an above-the-line adjustment (AGI is $61,592.09). It then used a $14,600 standard deduction instead of $15,750, which gave $949.80."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,530 equals 2.5% of about $61,200, roughly the full $61,592 AGI, so it skipped the $15,750 standard deduction that brings taxable income down to $45,842.09."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,700 standard deduction instead of Arizona's $15,750. Taxable income came out $50 too high ($45,892) and tax was $1,147.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its taxable income of $46,092 implies a $15,500 standard deduction instead of $15,750. That gave $1,152.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona standard deduction instead of $15,750. That gave taxable income of $46,592 and tax of $1,164.80."
-us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used a $15,400 standard deduction instead of $15,750 and computed 2.5% × $46,192 = $1,154.80. It then misplaced the decimal ('$1,1548.0') and submitted $11,548, a tenfold error."
-us,scenario_018,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona standard deduction instead of $15,750. That overstated taxable income at $46,592 and gave $1,165 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $1,941 is more than 2.5% of total gross income ($68,055 × 2.5% = $1,701), so it applied a rate above Arizona's flat 2.5%. The correct figure is 2.5% on $45,842.09 of taxable income after the $15,750 standard deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It set Arizona tax to $0 without any computation. It ignored the $45,842.09 of taxable income that remains after the $15,750 standard deduction and is taxed at the flat 2.5% rate for $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $18,500 of medical and health-insurance deductions by counting the two $9,000 premium fields as separate premiums. Arizona itemized deductions are only $10,786.71 ($9,500 medical plus $1,286.71 in state and local taxes), so the $15,750 standard deduction applies, not its $18,500."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the federal 2026 standard deduction of $16,100 instead of Arizona's $15,750. That understated taxable income at $45,492 and gave $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction instead of Arizona's $15,750 from $61,592 of AGI. That gave $1,137.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its taxable income of $45,492 reflects the federal $16,100 standard deduction instead of Arizona's $15,750. That understated tax at $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used the federal 2026 single standard deduction of $16,100 instead of Arizona's $15,750. That gave taxable income of $45,492 instead of $45,842.09 and tax of $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $45,492 of taxable income comes from the federal $16,100 standard deduction instead of Arizona's $15,750. That gave $1,137.30 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It treated $16,100 as the Arizona standard deduction, but the Arizona 2026 single amount is $15,750. Its taxable income was $45,492 and its tax $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied 'progressive rates averaging under 3%' to about $61,000, even though Arizona has used a flat 2.5% rate since 2023. It also never subtracted the $15,750 standard deduction, which brings taxable income to $45,842.09."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It projected a $15,450 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,142 and tax of $1,154."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction instead of $15,750. That overstated taxable income at $46,192 and gave $1,155 instead of $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It conformed Arizona to the federal 2026 $16,100 standard deduction, but Arizona's 2026 amount is $15,750. Its taxable income was $45,492 and its tax $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $15,375 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,217 and tax of $1,155."
-us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of $15,750 and also subtracted a $2,300 personal exemption. Arizona has no taxpayer personal exemption, so its taxable income of $43,192 is too low and gave $1,080."
-us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is nothing to score against $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona single standard deduction instead of $15,750. That gave taxable income of $46,592 and tax of $1,164.80."
-us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with no reasoning. That ignores the $45,842.09 of Arizona taxable income ($61,592.09 AGI minus the $15,750 standard deduction), which the flat 2.5% rate taxes at $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the federal 2026 $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30."
-us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left the $6,174 traditional 401(k) and the $289 traditional IRA in income, using AGI of $68,055 instead of $61,592.09. It also used a $14,600 standard deduction instead of $15,750, which inflated taxable income to $53,455 and gave $1,336."
-us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $14,508 standard deduction instead of $15,750. It then applied an effective rate of about 1.83% ($832 on $45,486) instead of Arizona's flat 2.5% rate, which on $45,842.09 gives $1,146.05."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the un-indexed $15,750 standard deduction instead of the 2026 indexed $16,100. It also subtracted the $1,625 federal car-loan interest deduction, which is a federal below-the-line deduction and not part of the Arizona AGI computation. It then submitted $1,503, which matches none of its own intermediate results ($1,146 or $1,105)."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied repealed graduated brackets (2.55%/3.34%) instead of Arizona's flat 2.5% rate and used a $14,600 standard deduction instead of $16,100. It also got the AGI arithmetic wrong ($61,567 instead of $61,592), then inflated the result to an unsupported $2,175."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It guessed Arizona taxable income at about $44,000, which implies a deduction near $17,600. The correct figure is $61,592.09 minus the $16,100 single standard deduction, or $45,492.09, and 2.5% of that is $1,137.30, not $1,100."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the AGI ($61,592) and the 2.5% rate right but used a guessed $15,000 standard deduction instead of the 2026 indexed $16,100. That overstated taxable income by $1,100 and the tax by about $28."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the $14,600 2024 federal standard deduction instead of Arizona's 2026 $16,100 to reach $1,175. It then raised the answer to $1,471 with no stated basis, even though a larger deduction would lower the tax, not raise it."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning correctly applied the $16,100 standard deduction to $61,592 AGI and reached $1,137 at 2.5%. It then submitted $1,481, a number that contradicts its own final computation."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $14,600 standard deduction carried over from an older federal value instead of Arizona's 2026 single deduction of $16,100. That overstated taxable income by $1,500 and gave $1,175 instead of $1,137.30."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the stale $14,600 standard deduction to reach $1,175, then raised the answer to $1,450 for invented 'HSA-like premium considerations'. Neither step reflects the correct $16,100 deduction at 2.5%."
+us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction, a pre-HB 4168 projection, instead of the $16,100 Arizona single deduction for 2026. That left taxable income $700 too high."
+us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut federal AGI to $52,592 by wrongly excluding $9,000 of health premiums from wages, when the AGI is $61,592.09. It also used a $7,700 standard deduction, which is half the single amount, instead of $16,100. The two errors partly offset and gave $1,122.30."
+us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction instead of the 2026 indexed $16,100, which overstated taxable income by $450 and gave $1,148.55."
+us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It correctly used the $16,100 standard deduction but also subtracted a $2,100 'personal exemption'. Arizona has no general personal exemption; the $2,100 exemption applies only to filers aged 65 or older or blind, and this filer is 53. The extra exemption understated the tax by $52.50."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It projected a $14,600 standard deduction instead of Arizona's 2026 single deduction of $16,100, which overstated taxable income by $1,500 and gave $1,174.80."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It showed no computation. $1,400 at 2.5% implies about $56,000 of taxable income, so it subtracted far less than the $16,100 standard deduction from the $61,592 AGI; the correct taxable income is $45,492.09."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $13,790 standard deduction instead of Arizona's 2026 single amount of $16,100, which overstated taxable income by $2,310 and gave $1,195."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $52,592 by wrongly removing $9,000 of health premiums from income, when the AGI is $61,592.09. It also used a $14,600 standard deduction instead of $16,100. Both errors together gave $949.80."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It showed no computation. $1,530 at 2.5% equals about $61,200 of taxable income, which is essentially the full AGI, so it effectively skipped the $16,100 standard deduction."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction instead of the 2026 indexed $16,100, which overstated taxable income by $400 and gave $1,147.30."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $46,092 taxable income implies a $15,500 standard deduction instead of the $16,100 Arizona single deduction for 2026, which overstated the tax by $15."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of the 2026 indexed $16,100, which overstated taxable income by $1,100 and gave $1,164.80."
+us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used a $15,400 standard deduction instead of $16,100. It then misplaced the decimal point, writing '$1,1548.0' and submitting $11,548, ten times its own computed $1,154.80."
+us,scenario_018,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of Arizona's 2026 single deduction of $16,100, which gave $1,165 instead of $1,137.30."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It showed no computation. $1,941 is more than 2.5% of the household's entire gross income, so it applied a rate above Arizona's flat 2.5% or used old graduated brackets, and skipped the $16,100 standard deduction. The correct tax is 2.5% of $45,492.09."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed zero tax with no basis. Arizona taxes the $45,492.09 left after the $16,100 standard deduction at 2.5%, and no nonrefundable credit offsets that liability."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized by adding both $9,000 premium lines and $500 of medical costs into $18,500 of deductions. PolicyEngine's Arizona itemized deductions total only $10,637.30, so the $16,100 standard deduction applies and taxable income is $45,492.09, not $43,092."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied 'progressive rates' to about $61,000 of income. Arizona has used a flat 2.5% rate since 2023, and the $16,100 standard deduction lowers taxable income to $45,492.09."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a projected $15,450 standard deduction instead of the 2026 indexed $16,100, which overstated taxable income by $650 and gave $1,154."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used about $15,400 as the standard deduction instead of Arizona's 2026 single deduction of $16,100, which gave $1,155 instead of $1,137.30."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $15,375 standard deduction instead of the 2026 indexed $16,100, which overstated taxable income by $725 and gave $1,155."
+us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly used the $16,100 standard deduction but also subtracted a made-up ~$2,300 exemption. Arizona has no personal exemption for a single filer under 65 who is not blind, so taxable income is $45,492.09, not $43,192."
+us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or reasoning for state_income_tax_before_refundable_credits, so its output for this variable is missing. The correct value is $1,137.30: 2.5% of $61,592.09 minus the $16,100 standard deduction."
+us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used a $15,000 Arizona standard deduction instead of the 2026 indexed $16,100, which gave taxable income of $46,592 instead of $45,492.09 and tax of $1,164.80."
+us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned zero with no reasoning. Arizona's flat 2.5% tax applies to the $45,492.09 left after the $16,100 standard deduction, which gives $1,137.30, and no nonrefundable credit removes it."
+us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It assumed gross wages already excluded the traditional 401(k) and ignored the IRA deduction, which overstated AGI at $68,055 instead of $61,592.09. It also used a $14,600 standard deduction instead of $16,100."
+us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $14,508 standard deduction instead of $16,100. It then applied an invented 'rate structure' that taxes $45,486 at about 1.8%, when Arizona's flat rate is 2.5%."
us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"It counted only the $13,834 real estate tax as SALT and so used the $16,100 standard deduction, for taxable income of $284,050. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax raises itemized SALT to $17,090.34, and the extra ~$990 of taxable income at 35% accounts for its ~$347 gap to that reference. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% is nearly all of its $69.27 gap to the $68,334.73 the conventions give with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 nonitemizer charitable deduction, for taxable income of $283,854. Under the release's sales tax convention (c_irs_sales_tax_2025) that is the household's choice, because the IRS 2025 table leaves SALT below the standard deduction; the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84, and the $794 difference at 35% explains its $278 gap to that reference. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"It subtracted the $138,375 of unreimbursed employee business expenses above the line, but those are nondeductible miscellaneous itemized deductions under current law. It also used a $15,000 standard deduction. It then submitted $66,661, a figure disconnected from its own $27,268 bracket computation, instead of taxing $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), and adding NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 of wages, although gross wages already include overtime. It capped SALT at $10,000 instead of the $40,400 2026 cap and ignored the sales tax table deduction; neither changes its $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but it left out the $196 nonitemizer charitable deduction. It also omitted NIIT and then submitted $70,813, which does not match its own $82,743 computation."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"It disallowed the alimony deduction and invented $23,855 of mortgage interest from the loan balance, even though no interest was listed. It capped SALT at $10,000 and used 2025 brackets. It then submitted $71,866, which does not match its own ~$64,291 sum."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"It guessed itemized deductions of $30,000–40,000 with no basis. Itemized deductions are SALT alone: $13,834 of real estate tax plus the sales tax table amount, which the frozen reference projects for $17,090.34 in all; the release's sales tax convention (c_irs_sales_tax_2025) takes the IRS 2025 table, whose $1,595 for this filer leaves SALT below the $16,100 standard deduction. Its $72,700 answer is an unsupported estimate rather than tax on $283,854.19 of taxable income under the convention ($283,059.84 in the frozen reference) plus $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 nonitemizer charity deduction, for taxable income of $283,854. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction even with Texas's general sales tax deduction added to the $13,834 of property tax; only the frozen reference's projected sales tax brings SALT to $17,090.34 and lowers taxable income to $283,059.84. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and capped SALT at $10,000 instead of $40,400. It used a projected $15,400 standard deduction and stale bracket projections instead of the $16,100 deduction and actual 2026 brackets. It also left out the $222.64 NIIT, and its $70,398 reflects all of these errors."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"It used the 2025 brackets and $15,000 standard deduction instead of 2026 values and capped SALT at $10,000 instead of $40,400. It also added $900 of Additional Medicare Tax, which is a payroll tax and not part of income tax. It then inflated its own $70,467 to an arbitrary $76,500."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"It treated alimony as nondeductible, which left AGI at $305,859 instead of $300,150. It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the better choice), but not the $196 nonitemizer charitable deduction. Its taxable income of $289,759 is $5,904.81 above the convention's $283,854.19 and $6,699.16 above the frozen reference's $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"It applied pre-TCJA sunset law, deducting $132,258 of unreimbursed employee expenses above a 2% floor and claiming a personal exemption. OBBBA permanently eliminated both, so the correct figure is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"It applied pre-TCJA rules, including miscellaneous itemized deductions for employee expenses, a personal exemption, and 15/25/28% brackets, even though OBBBA made the TCJA structure permanent. It also subtracted the ESI premiums from the listed gross wages and dropped the alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"It took a $25,000 overtime deduction without applying the single-filer $12,500 cap and the $100-per-$1,000 phaseout above $150,000 MAGI. That phaseout eliminates the deduction entirely at $300k. It also dropped the alimony deduction and omitted the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"It deducted $132,258 of unreimbursed employee expenses as miscellaneous itemized deductions and claimed a $5,450 personal exemption under pre-TCJA brackets, although OBBBA permanently suspended both. It also omitted the alimony deduction and NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"It assumed TCJA had expired, allowed miscellaneous itemized employee expenses and a personal exemption, and then substituted an AMT computation. It also subtracted the ESI premiums from AGI. Under current law, regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or on $283,059.84 in the frozen reference, which itemizes a projected sales tax."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"It used AGI of $305,859, ignoring the $5,709 alimony deduction, and capped SALT at $10,000 instead of the 2026 $40,400 cap. Not itemizing is still the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction under the $40,400 cap; only the frozen reference's projected sales tax lifts real estate plus sales tax to $17,090.34, above it."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"It assumed the TCJA individual provisions expire in 2026, deducted employee business expenses over the 2% floor, and then applied AMT. OBBBA made the suspension of miscellaneous itemized deductions permanent, so the answer is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"It subtracted the ESI premiums from AGI and itemized $146,570, including the permanently suspended miscellaneous employee expenses. It then relied on an AMT computation instead of regular tax on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), or $283,059.84 in the frozen reference, which itemizes a projected sales tax, plus NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"It skipped the alimony deduction, used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax), and left out NIIT. Its $50,568 is far below even the bracket tax on its own taxable income, which shows a computation error."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"It applied pre-TCJA law: uncapped SALT, miscellaneous employee expenses above the 2% floor, and personal exemptions. It also subtracted the ESI premiums from wages. Current law permanently disallows the employee expenses and exemptions, so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax)."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"It assumed the TCJA sunset and itemized the unreimbursed employee expenses with a personal exemption. It also subtracted the ESI premiums and dropped alimony. Under permanent OBBBA law those employee expenses are nondeductible, and taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax)."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"It omitted the $5,709 alimony deduction and used the 2025 $15,000 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT, property plus a projected sales tax table amount). It also left out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"It treated 2026 as a TCJA-sunset year, deducted $132,372 of employee business expenses as miscellaneous itemized deductions, and took a personal exemption. OBBBA permanently suspends both. It also omitted NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"It declared the alimony expense nondeductible, leaving AGI at $305,859 instead of $300,150. It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax lifts itemized SALT to $17,090.34, above $16,100), but not the $196 nonitemizer charitable deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"It treated the $138,375 of unreimbursed employee expenses and unlisted mortgage interest as deductions that wipe out all tax. Employee expenses are nondeductible under current law, and no interest was given. Correct itemized deductions are only SALT, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction leaves $68,112.09 of regular tax plus NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its ~$31,472 answer matches pre-TCJA treatment that deducts the unreimbursed employee business expenses as miscellaneous itemized deductions. Those are permanently disallowed. Itemized deductions are SALT alone, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction gives tax of $68,112.09 plus $222.64 NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834.07 plus NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"It used the $16,100 standard deduction, for taxable income of $284,050, because it compared only the $13,834 of property tax against it. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% matches its $68.46 gap to the $68,334.73 the conventions give with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from the listed gross wages, which understates income. It used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax), but not the $196 nonitemizer charitable deduction, and it omitted the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT (property plus a projected sales tax). It also left out the $222.64 net investment income tax, which the frozen reference includes in this total; its $68,112.00 is $0.09 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"Its $70,179 matches tax on $289,759 of taxable income. That figure ignores the $5,709 alimony deduction and uses the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. It also omits the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"It used the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference's projected sales tax makes itemized SALT of $17,090.34 larger. It also left out the $222.64 NIIT, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"It computed tax on taxable income of $289,759, which drops the $5,709 above-the-line alimony deduction. It also takes the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of real estate plus a projected sales tax."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"It started from $305,859 of gross income with no alimony deduction and used the $16,100 standard deduction. The alimony deduction lowers AGI to $300,150, and the nonitemizer charitable deduction adds $196 to the standard deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's itemized SALT of $17,090.34 (including a projected sales tax table amount) exceeds it."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. It also omitted the $222.64 net investment income tax, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"It gave a rounded guess of ~$65k with no computation. It did not deduct alimony, add the $196 nonitemizer charitable deduction to its standard deduction, tax the resulting $283,854.19 at 2026 brackets, or add the $222.64 NIIT; its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax and taxes $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"It applied pre-TCJA law, deducting $132,372 of miscellaneous employee expenses and a personal exemption under reverted brackets. OBBBA permanently suspended both, so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or $283,059.84 in the frozen reference, which itemizes a projected sales tax, not $148,321."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"It assumed the TCJA sunset, deducted the employee expenses and uncapped SALT, and then fell back to AMT. Under current permanent law the employee expenses are nondeductible and regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025) or $283,059.84 in the frozen reference, which itemizes a projected sales tax."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"It used the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the larger deduction), but it left out the $196 nonitemizer charitable deduction, for taxable income of $284,050 instead of the convention's $283,854.19. It also omitted the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,It applied pre-TCJA brackets and deducted the unreimbursed employee expenses above a 2% floor. It also subtracted the ESI premiums from wages and dropped alimony. OBBBA permanently disallows miscellaneous itemized deductions.
-us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from gross wages, took the $196 of charity above the line rather than as the nonitemizer charitable deduction, which lowers taxable income by the same $196, and skipped the $5,709 alimony deduction. Its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. Offsetting errors landed it at $67,479."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was no answer to score."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"It treated the itemized amount as $13,834 without the sales tax and took the standard deduction plus the $196 nonitemizer charity deduction, which is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025): the IRS 2025 table leaves SALT below the standard deduction either way. Only the frozen reference's projected sales tax raises itemized SALT to $17,090.34 and lowers taxable income to $283,059.84; its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"It submitted $0 with no reasoning. The correct derivation under the release's sales tax convention (c_irs_sales_tax_2025) is AGI of $300,150 after alimony, less the $16,100 standard deduction and the $196 nonitemizer charitable deduction, for taxable income of $283,854.19, tax of $68,112.09 and NIIT of $222.64; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $283,059.84 and $67,834.07. A zero is consistent only with treating the $138,375 of employee expenses as fully deductible."
-us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"It declared alimony nondeductible, for AGI of $305,859, and took the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's projected sales tax, added to property tax, makes itemizing larger. It also omitted the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"It added the $42,857 overtime premium on top of the $300,000 of gross wages, capped SALT at $10,000 and used outdated brackets. It then submitted $68,513.50, which contradicts its own computed $85,241.65."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"It applied married-filing-jointly brackets, a qualified business income deduction and a child tax credit to a single, childless wage earner with no business income. None of those apply."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"It counted only the $13,834 of real estate taxes as SALT and took the $16,100 standard deduction. It never added the Texas general sales tax deduction from the IRS optional table (~$3,256), which brings itemized SALT to $17,090 and makes itemizing better. Taxable income came out at $284,050 instead of $283,060, so its answer is about $347 too high."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction. It missed the optional-table sales tax deduction available to a Texas filer, which raises SALT to $17,090. With that, the filer itemizes, the $196 gift falls below the 0.5%-of-AGI floor, and taxable income is $283,060 rather than $283,854."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"It subtracted the $138,375 of unreimbursed employee business expenses above the line, but these remain nondeductible miscellaneous itemized deductions. It also used a $15,000 standard deduction. Its final $66,661 has no connection to its own $27,268 computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,other,False,"It admitted it never completed the computation and submitted 0 as a placeholder. The correct derivation (AGI $300,150, $17,090 itemized SALT, taxable income $283,060, plus NIIT) gives $68,056.71."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 of gross wages, which already include it. It capped SALT at $10,000 instead of the 2026 $40,400 cap and ignored the Texas sales tax deduction. It also omitted NIIT, and its reported $70,813 contradicts its own $82,743 computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"It made up about $23,855 of mortgage interest from the loan balance, although no interest paid was listed. It disallowed the alimony deduction and capped SALT at $10,000 rather than the 2026 $40,400 cap. Its reported $71,866 also does not match its own ~$64,291 total."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"It guessed itemized deductions of $30,000–40,000 without itemizing anything. The actual deductions are $13,834 of property tax plus ~$3,256 of table sales tax ($17,090). Its $72,700 is a figure with no derivation behind it."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"It compared only the $13,834 of property tax with the $16,100 standard deduction and left out the optional-table general sales tax deduction a Texas filer can claim. That deduction raises SALT to $17,090 and makes itemizing better. Taking the standard deduction plus the $196 non-itemizer charity left taxable income at $283,854 instead of $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction, capped SALT at $10,000, and skipped the Texas sales tax deduction. It projected 2025 brackets and a $15,400 standard deduction instead of the 2026 OBBBA values ($16,100), and it left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"It used 2025 brackets with a $15,000 standard deduction and capped SALT at $10,000, which made it miss that $17,090 of SALT (including table sales tax) beats the $16,100 deduction. It added the $900 Additional Medicare Tax, which is a payroll tax, to income tax. It then raised its own $70,467 to an arbitrary $76,500."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"It treated the $5,709 of alimony as nondeductible, putting AGI at $305,859 instead of $300,150. It also took the standard deduction instead of itemizing $17,090 of SALT (property tax plus Texas optional-table sales tax), so it taxed $289,759 instead of $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"It assumed TCJA had expired in 2026. On that basis it deducted the $138,375 of unreimbursed employee expenses above a 2% floor and claimed a personal exemption. OBBBA made both the miscellaneous-deduction suspension and the zero personal exemption permanent, so taxable income is $283,060, not ~$155,500."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"It applied pre-TCJA law: a 2%-floor deduction for employee expenses, a personal exemption, and 15/25/28% brackets, all of which OBBBA permanently replaced. It also subtracted the $8,389 of ESI premiums from the gross wages, which are already given."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"It took a $25,000 overtime deduction, but that is the joint cap. A single filer is capped at $12,500, and the cap phases out completely at $100 per $1,000 of MAGI over $150,000. It also skipped the alimony deduction, used a $15,350 standard deduction instead of itemizing $17,090 of SALT, and left out NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"It used pre-TCJA 2026 law, deducting employee business expenses above a 2% floor and claiming a personal exemption. OBBBA permanently eliminated both, so taxable income is $283,060, not $154,121."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"It modeled a TCJA sunset with miscellaneous itemized deductions, a personal exemption, and an AMT recapture, all of which OBBBA removed. It also subtracted ESI premiums from wages and skipped the alimony deduction. The correct path is to itemize $17,090 of SALT against AGI of $300,150."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"It applied a $10,000 SALT cap instead of the 2026 $40,400 cap, skipped the alimony deduction, and left out the Texas sales tax deduction. As a result it took the standard deduction instead of itemizing $17,090 against AGI of $300,150."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"It assumed TCJA had expired and allowed the unreimbursed employee expenses as a 2%-floor miscellaneous deduction, which then triggered AMT. OBBBA permanently disallows those expenses, so regular tax on $283,060 plus NIIT applies. It also subtracted ESI premiums from wages."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"It itemized $146,570, which includes the suspended employee business expenses, under an assumed TCJA sunset with AMT. It also subtracted ESI premiums from the gross wages. The correct itemized total is only the $17,090 of SALT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"It skipped the alimony deduction, took the standard deduction instead of itemizing $17,090 of SALT, and left out NIIT. Its $50,568 is far below what the 2026 single brackets (35% above $256,225) produce on ~$290,000 of taxable income, so it misapplied the bracket schedule."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"It applied pre-TCJA rules: uncapped SALT, a 2%-floor deduction for the $138,375 of employee expenses, and personal exemptions. OBBBA permanently disallows employee expenses and exemptions. It also subtracted ESI premiums from wages."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"It assumed a TCJA sunset and deducted unreimbursed employee expenses above a 2% floor plus a personal exemption, both permanently eliminated by OBBBA. It also subtracted the $8,389 of ESI premiums from the gross wages."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"It skipped the alimony deduction and used a $15,000 standard deduction with 2025 brackets instead of the 2026 values. It missed the Texas sales tax deduction that makes itemizing $17,090 better, and it left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"It treated 2026 as a post-TCJA reversion year and deducted $132,372 of employee business expenses plus a personal exemption. OBBBA made the suspension of both permanent, so taxable income is $283,060, not ~$148,748."
+us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and counted only $13,834 of property tax plus $196 of charity as itemized deductions. It never added the Texas optional-table sales tax, which raises SALT to $17,090, so it wrongly took the $16,100 standard deduction on taxable income of $289,759."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"It claimed deductions wipe out all tax, relying on unlisted mortgage interest and the $138,375 of unreimbursed employee expenses. Those expenses are permanently nondeductible, and no interest paid was given. Itemized deductions total only $17,090, which leaves $283,060 taxable."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its ~$31,472 matches a pre-TCJA calculation that deducts the $138,375 of employee business expenses as a miscellaneous itemized deduction. OBBBA permanently disallows that deduction, so the correct taxable income is $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"It took the $16,100 standard deduction without counting the Texas optional-table general sales tax deduction (~$3,256). With it, SALT is $17,090 and itemizing is better. Taxable income came out $990 too high ($284,050 instead of $283,060)."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"It subtracted the $8,389 of employer-sponsored insurance premiums from the given gross wages and left out the $222.64 NIIT. It also took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas table sales tax."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 non-itemizer charity deduction, missing that property tax plus Texas optional-table sales tax ($17,090) makes itemizing better. It also left out the $222.64 net investment income tax that belongs in the total."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"It skipped the $5,709 alimony deduction and took the standard deduction instead of itemizing $17,090 of SALT, which includes the Texas sales tax table amount. It also left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"It used the $16,100 standard deduction plus the $196 non-itemizer charitable deduction instead of itemizing $17,090 of SALT (property tax plus Texas optional-table sales tax). It also left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"It skipped the $5,709 above-the-line alimony deduction and took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas general sales tax. It therefore taxed about $289,759 instead of $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"It started from $305,859 without deducting the $5,709 of alimony and applied the $16,100 standard deduction. It missed that property tax plus the Texas optional-table sales tax ($17,090) makes itemizing better, so it taxed $289,759 instead of $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"It chose the standard deduction plus the $196 non-itemizer charity deduction instead of itemizing $17,090 of SALT, which includes the Texas sales tax table amount. It also left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"It gave a rounded $65,000 guess based on the standard deduction, with no computation. It skipped the alimony deduction, the itemized SALT with Texas sales tax, the 2026 brackets, and NIIT, which together yield $68,056.71."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"It applied pre-TCJA 2026 law, deducting $132,372 of employee business expenses above a 2% floor and claiming a personal exemption. OBBBA permanently disallows both, so taxable income is $283,060, not $148,321."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"It assumed a TCJA sunset in which employee expenses and uncapped real estate taxes are deductible and then added back under AMT. OBBBA keeps employee expenses permanently nondeductible, so regular tax on $283,060 applies. It also skipped the alimony deduction."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"It took the $16,100 standard deduction without adding the Texas optional-table general sales tax to the $13,834 of property tax. That combined $17,090 makes itemizing better. It also left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,"It used projected pre-TCJA 10/15/25/28% brackets and deducted the $138,375 of employee expenses above a 2% floor. OBBBA permanently eliminated that deduction and kept the TCJA brackets. It also subtracted ESI premiums from the gross wages."
+us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from the gross wages, treated the $196 of charity as an above-the-line deduction, and skipped the $5,709 alimony deduction. It then took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas table sales tax."
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so the output is missing."
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"It compared only the $13,834 of property tax with the $16,100 standard deduction and left out the Texas optional-table sales tax, which raises SALT to $17,090 and makes itemizing better. Taking the standard deduction plus the $196 non-itemizer charity left taxable income at $283,854 instead of $283,060."
+us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"It submitted 0 with no reasoning, which means it computed no tax at all. The correct calculation is AGI of $300,150 less $17,090 of itemized SALT, giving $283,060 taxable, with $67,834 of tax plus $222.64 of NIIT, for $68,056.71."
+us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and counted only property tax plus charity ($14,030) as itemized deductions, never adding the Texas optional-table sales tax that raises SALT to $17,090. It also left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"It added the $42,857 overtime premium on top of the $300,000 of gross wages, which already include it. It capped SALT at $10,000 instead of $40,400 and used a $15,750 standard deduction. Its reported $68,513.50 contradicts its own $85,241.65 computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"It used married-filing-jointly brackets for a single filer, took a QBI deduction with no business income, and subtracted a child tax credit for a household with no children."
us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_020,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model derived the reference exactly ($11,439 + $4,350 + $900 = $16,689) and then overrode it by deducting the $8,389 employer-sponsored insurance premium from Medicare wages, dropping to $16,491.86; PolicyEngine's payroll tax base is the stated gross wage of $300,000, unreduced by ESI premiums. It then submitted $15,200, a figure matching neither its correct derivation nor its own revised $16,491.86."
-us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500, added the $42,857 FLSA overtime premium on top of the $300,000 gross wage even though the prompt states gross wages already include overtime, and further deducted the $8,389 ESI premium from FICA wages. After cycling through $15,703.20, $16,710.34, and $16,513.19 it submitted $23,119.50, which none of its own calculations produce."
-us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model computed the reference stack correctly at $16,689 using the $184,500 wage base, then second-guessed itself with the stale $176,100 and an invented $183,600 base, and finally submitted $13,283.40 - a value produced by none of the three derivations it wrote. The failure is at the reporting step: it abandoned a correct $11,439 + $4,350 + $900 computation for an unanchored number."
-us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model applied the 2025 wage base of $176,100 rather than the 2026 base of $184,500, yielding $10,918.20 + $4,350 + $900 = $16,168.20 in its own reasoning, then submitted $6,219 - a number inconsistent with the total it had just computed. Both errors compound: a $520.80 wage-base shortfall plus a $9,949 break between its arithmetic and its submission."
-us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"The model derived the reference exactly - $11,439 (6.2% of $184,500) + $4,350 + $900 = $16,689 - and then discounted its own correct result for 'base uncertainty,' submitting $15,866. The 2026 Social Security wage base is $184,500, so its first computation was the answer and the downward adjustment introduced the entire $823 error."
-us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model carried the 2025 Social Security wage base of $176,100 into 2026, explicitly reasoning that the base 'is expected to be similar,' when the 2026 base is $184,500. Its Medicare ($4,350) and Additional Medicare ($900) components are exact, so the entire $520.80 shortfall is 6.2% of the $8,400 of wage base it omitted."
-us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model first used the correct 2026 wage base of $184,500 and reached $16,689, then replaced it with an invented $182,600 base to get $11,321.20 of Social Security tax and submitted $16,571.20. The $117.80 error is exactly 6.2% of the $1,900 of wage base it discarded from its own correct figure."
-us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model estimated the 2026 Social Security wage base at $183,700 rather than the actual $184,500, producing $11,389 of Social Security tax instead of $11,439. Medicare ($4,350) and Additional Medicare ($900) are exact, so the whole $50 error is 6.2% of the $800 wage-base understatement."
-us,scenario_020,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model used a $184,200 Social Security wage base instead of the 2026 base of $184,500, giving $11,420.40 rather than $11,439. Every other component - $4,350 Medicare, $900 Additional Medicare, no Texas employee payroll tax - is correct, so the $18.60 miss is exactly 6.2% of the $300 of missing wage base."
-us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model made two base errors: it applied the 2025 wage base of $176,100 instead of the 2026 base of $184,500 (costing $520.80), and it reduced Medicare wages from $300,000 to $291,611 by subtracting the $8,389 ESI premium (costing $121.64 of Medicare tax and $75.50 of Additional Medicare Tax). PolicyEngine computes all three components on the unreduced $300,000 gross wage."
-us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model named no wage base and landed $1,173 below the reference; the correct derivation is $184,500 x 6.2% = $11,439 plus $300,000 x 1.45% = $4,350 plus $100,000 x 0.9% = $900. Its $15,516 is consistent with carrying a pre-2026 Social Security cap near $168,600 while also shaving the $8,389 ESI premium off Medicare wages - neither of which applies in 2026 on this gross wage."
-us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model explicitly computed Medicare on 'gross wages minus ESI,' i.e. $291,611 instead of $300,000, and used a Social Security cap of roughly $183,600 rather than the 2026 base of $184,500. Employer-sponsored insurance premiums do not reduce the payroll tax base in this computation, and the two errors together produce the $253 shortfall to $16,436."
-us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model applied 6.2% to a wage cap near $177,300 (yielding about $10,993 instead of $11,439) and computed both the 1.45% Medicare tax and the 0.9% surtax on ESI-reduced wages of $291,611 rather than the full $300,000. The payroll base is gross wages of $300,000 and the 2026 cap is $184,500, leaving its $16,045 short by $644."
-us,scenario_020,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model stated the correct rate structure but never applied the Social Security cap in its arithmetic; its $22,864 is consistent with charging 6.2% on wages far above the 2026 base of $184,500 (near the uncapped $300,000, which gives $18,600) on top of the $4,350 Medicare tax. Capping Social Security wages at $184,500 gives $11,439 and a total of $16,689, so the missing cap accounts for the entire $6,175 overstatement."
-us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used an invented $181,800 Social Security cap instead of the 2026 base of $184,500 ($11,271.60 rather than $11,439) and computed Medicare and the 0.9% surtax on $291,611 after deducting the $8,389 ESI premium. Both Medicare taxes run on the unreduced $300,000 wage, so the correct components are $4,350 and $900, not $4,228.36 and $824.50."
-us,scenario_020,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model's $16,324.46 decomposes as 6.2% on a $181,800 cap ($11,271.60), 1.45% on ESI-reduced wages of $291,611 ($4,228.36), and 0.9% on $91,611 ($824.50). The 2026 wage base is $184,500 and the payroll base is the full $300,000 of gross wages, so all three of its components are understated."
-us,scenario_020,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model explicitly capped Social Security wages at $176,100, the 2025 base, instead of the 2026 base of $184,500, giving $10,918.20 rather than $11,439. Its $4,350 Medicare and $900 Additional Medicare figures are exact, so the $520.80 gap is entirely 6.2% of the $8,400 of indexation it missed."
-us,scenario_020,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model projected the 2026 Social Security wage base at $180,000, producing $11,160 of employee Social Security tax instead of $11,439 on the actual $184,500 base. Medicare ($4,350) and the Additional Medicare Tax ($900) are correct, so the $279 shortfall is 6.2% of the $4,500 of wage base it left out."
-us,scenario_020,payroll_tax,glm-5.3,llm_error,payroll_tax_base,False,"The model set the 2026 wage base at $181,800 rather than $184,500, yielding $11,271.60 of Social Security tax instead of $11,439. It correctly recognized that the $42,857 overtime premium sits inside the $300,000 gross wage and that Texas has no employee payroll tax, so the entire $167.40 error is the $2,700 wage-base shortfall at 6.2%."
-us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model described the correct three-component structure but showed no arithmetic and overshot by $1,659; the correct components are $184,500 x 6.2% = $11,439, $300,000 x 1.45% = $4,350, and $100,000 x 0.9% = $900. Backing out its $4,350 and $900, its $18,348 implies Social Security tax of $13,098 - 6.2% of about $211,300, far above the 2026 cap of $184,500."
-us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted that no Additional Medicare Tax was triggered, but $300,000 of wages exceeds the $200,000 single-filer threshold by $100,000, generating $900 of 0.9% surtax. Its $11,689 is essentially the Social Security component alone, omitting both that $900 surtax and the $4,350 of 1.45% Medicare tax on the full wage."
-us,scenario_020,payroll_tax,grok-4.3,llm_error,other,False,"The model's stated components - 6.2% on a $176,100 base, 1.45% on $300,000, and 0.9% on $100,000 - sum to $16,168.20, yet it submitted $10,578. It compounded a stale 2025 wage base (the 2026 base is $184,500, giving $11,439) with a reporting break that discards its own arithmetic by $5,590."
-us,scenario_020,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model estimated the 2026 Social Security wage base at $183,144 instead of $184,500, producing $11,355 of Social Security tax rather than $11,439. Its Medicare ($4,350) and Additional Medicare ($900) components and its Texas no-state-payroll-tax finding are correct, so the $84 gap is 6.2% of the $1,356 wage-base understatement."
-us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model explicitly set 'FICA wages 291611 after health premiums,' subtracting the $8,389 ESI premium from the Medicare and surtax base, and used a $184,000 cap instead of the 2026 base of $184,500. Both Medicare taxes apply to the full $300,000 of gross wages ($4,350 and $900), so its $4,228 and $825 are understated."
-us,scenario_020,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model got the 2026 wage base exactly right ($184,500 x 6.2% = $11,439) but then computed Medicare on $291,611 after deducting the $8,389 employer-sponsored insurance premium, and applied the 0.9% surtax to only $91,611 above the $200,000 threshold. The payroll tax base is the unreduced $300,000 gross wage, so its $197 shortfall is precisely 2.35% of the $8,389 it removed."
-us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the submission carries no payroll tax figure at all. The required derivation is $11,439 of employee Social Security tax on the $184,500 2026 wage base, $4,350 of Medicare tax on $300,000, and $900 of Additional Medicare Tax on the $100,000 above the $200,000 single threshold."
-us,scenario_020,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with no derivation, treating a household with $300,000 of wage income as owing no employee-side payroll tax. Wages generate 6.2% Social Security tax up to the $184,500 2026 cap ($11,439), 1.45% Medicare tax on the full wage ($4,350), and the 0.9% surtax on the $100,000 over the $200,000 single threshold ($900); Texas's lack of an income tax does not exempt federal FICA."
-us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $42,857 FLSA overtime premium to the $300,000 gross wage to reach $342,857 of FICA wages, double-counting overtime the prompt states is already included in gross wages, and used the 2025 cap of $176,100 rather than the 2026 base of $184,500. It then submitted $19,436.31 rather than the $17,175.34 its own arithmetic produced."
-us,scenario_020,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model stated it included overtime premiums in wages, stacking the $42,857 FLSA premium on top of the already-inclusive $300,000 gross wage, and its $20,303 exceeds the reference by $3,614 - consistent with also running 6.2% well past the 2026 cap of $184,500. The correct components on the stated $300,000 wage are $11,439, $4,350, and $900."
+us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"It first reached the correct $16,689 ($11,439 + $4,350 + $900). It then wrongly subtracted the $8,389 ESI premiums from Medicare wages to get $16,491.86. Finally it submitted $15,200, which matches neither of its own totals."
+us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also added the $42,857 FLSA overtime premium on top of the $300,000 gross wages, which already include overtime, and subtracted the ESI premiums from FICA wages. Its submitted $23,119.50 then contradicts its own final sum of $16,513.19."
+us,scenario_020,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It listed the right components (capped Social Security, Medicare and Additional Medicare Tax) but never calculated them and submitted $0. The correct calculation gives $11,439 + $4,350 + $900 = $16,689."
+us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"It calculated the correct $16,689 using the $184,500 base, then tried other wage bases ($176,100 and $183,600). It submitted $13,283.40, which does not match any of its own calculations and is below even its lowest alternative of $16,168.20."
+us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, which understates Social Security tax by $520.80 and gives $16,168.20. It then submitted $6,219, which does not match its own total either."
+us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"It calculated the correct $16,689 from the $184,500 base, $4,350 Medicare and $900 Additional Medicare Tax. It then cut this to an arbitrary $15,866 to account for 'base uncertainty' instead of submitting its own calculation."
+us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried the 2025 wage base of $176,100 into 2026 instead of using the 2026 base of $184,500. That gives Social Security tax of $10,918.20 instead of $11,439, a total that is $520.80 too low."
+us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It first calculated the correct $16,689 with the $184,500 base, then switched to a made-up base of $182,600. That gives Social Security tax of $11,321.20 and a submitted total of $16,571.20."
+us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a guessed 2026 wage base of $183,700 instead of $184,500. That gives Social Security tax of $11,389 instead of $11,439, so its total is $50 short."
+us,scenario_020,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a wage base of $184,200 instead of the 2026 base of $184,500. That gives Social Security tax of $11,420.40 instead of $11,439."
+us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It used the 2025 wage base of $176,100 instead of $184,500. It also subtracted the $8,389 ESI premiums from Medicare wages, taxing $291,611 instead of $300,000, which lowered both Medicare tax ($4,228.36) and Additional Medicare Tax ($824.50)."
+us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"With $4,350 Medicare and $900 Additional Medicare Tax, its $15,516 total implies Social Security tax of about $10,266. That equals a wage base of about $165,600, far below the 2026 base of $184,500, so it applied an outdated cap."
+us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It explicitly reduced Medicare wages by the ESI premiums to $291,611, giving $4,228.36 Medicare tax and $824.50 Additional Medicare Tax. Its $16,436 total also implies a wage base of $183,600 instead of $184,500. Both choices lower the total."
+us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the $8,389 ESI premiums from Medicare wages ($291,611), even though the full $300,000 is subject to Medicare tax. It also used a wage base of about $177,300, giving Social Security tax of about $10,993 instead of $11,439."
+us,scenario_020,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"After $4,350 Medicare and $900 Additional Medicare Tax, its $22,864 total implies Social Security tax of about $17,614. That is consistent with applying 6.2% to nearly all wages without capping them at the $184,500 wage base, where the maximum tax is $11,439."
+us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It used a wage base of $181,800 instead of $184,500, giving Social Security tax of $11,271.60. It also subtracted the ESI premiums from Medicare wages ($291,611), which understates Medicare tax and Additional Medicare Tax by $121.64 and $75.50."
+us,scenario_020,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $16,324.46 total equals $11,271.60 + $4,228.36 + $824.50. That means it used a $181,800 wage base instead of $184,500 and wrongly subtracted the $8,389 ESI premiums from Medicare wages."
+us,scenario_020,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That gives Social Security tax of $10,918.20 instead of $11,439."
+us,scenario_020,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"It used a rounded estimate of $180,000 for the 2026 wage base instead of $184,500. That gives Social Security tax of $11,160 instead of $11,439."
+us,scenario_020,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"It used a wage base of $181,800 instead of the 2026 base of $184,500. That gives Social Security tax of $11,271.60 instead of $11,439, so its total is $167.40 short."
+us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $18,348 total is $1,659 above the maximum possible with a capped Social Security wage base, Medicare on $300,000, and Additional Medicare Tax on $100,000. That means it overstated the Social Security component beyond the 6.2% × $184,500 = $11,439 cap."
+us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It wrongly said no Additional Medicare Tax applied, even though wages exceed the $200,000 single threshold by $100,000 ($900 owed). Its $11,689 is also $4,100 below the correct $15,789 for Social Security plus regular Medicare."
+us,scenario_020,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of $184,500. It then added up its own components wrongly: $10,918.20 + $4,350 + $900 equals $16,168.20, not $10,578."
+us,scenario_020,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It used an estimated 2026 wage base of $183,144 instead of the published $184,500. That gives Social Security tax of $11,355 instead of $11,439."
+us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It reduced FICA wages to $291,611 by subtracting the ESI premiums, which understates Medicare tax and Additional Medicare Tax. It also used a $184,000 wage base instead of $184,500, giving Social Security tax of $11,408 instead of $11,439."
+us,scenario_020,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It used the correct $184,500 base ($11,439), but it subtracted the $8,389 ESI premiums from Medicare wages. That gave $4,228 Medicare and $825 Additional Medicare Tax instead of $4,350 and $900 on the full $300,000."
+us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score against the $16,689 total."
+us,scenario_020,payroll_tax,minimax-m3,llm_error,missing_output,False,"It submitted $0 with no calculation. It ignored the Social Security, Medicare and Additional Medicare Tax owed on $300,000 of wages, which total $11,439 + $4,350 + $900 = $16,689."
+us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 gross wages, even though gross wages already include overtime, which inflated Medicare and Additional Medicare Tax. It also used the 2025 base of $176,100 instead of $184,500. Its submitted $19,436.31 does not match its own sum of $17,175.34."
+us,scenario_020,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It said overtime premiums are included in the base, which is consistent with adding the $42,857 FLSA premium on top of gross wages that already include it. Even that double-count, with the correct cap, gives only about $17,696. Its $20,303 total therefore also overstates Social Security tax beyond the $11,439 maximum."
us,scenario_020,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no state individual income tax but then inserted an unexplained $2,380 nonrefundable state-tax amount. No state or local jurisdictional rule applies to create that liability for this Texas household, so the state-income-tax computation yields $0."
us,scenario_021,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_021,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model asserted aged/disabled eligibility after acknowledging roughly $27,508 of income, but never applied Missouri's ABD income test to establish qualification. Disability and assets below its assumed resource limit do not create eligibility when the person qualifies through no Medicaid category."
-us,scenario_021,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invoked general and medical-expense disregards without calculating countable income or showing that those deductions reduce income below Missouri's ABD limit. Age, disability, and resources do not substitute for satisfying the pathway's income requirement, and the resulting Medicaid category is NONE."
-us,scenario_021,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated Medicaid eligibility with an ABD spend-down or Medicare Savings Program pathway and incorrectly described $27,530 as low enough without applying a qualifying limit. It also treated Social Security as mostly excluded and assumed a favorable resource rule, neither of which establishes any PolicyEngine Medicaid category."
-us,scenario_021,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the head's income very low without applying a Missouri Medicaid eligibility pathway or threshold. The engine derives MAGI income of 1.72 times FPL and no qualifying category, so age and modest assets alone yield no Medicaid eligibility."
-us,scenario_021,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inferred eligibility directly from age and vaguely characterized income and resources as limited, without testing a specific Missouri category. The head qualifies through none of the Medicaid pathways, so those characteristics do not produce eligibility."
+us,scenario_021,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model assumed that being 85, disabled and on Medicare opens a Medicaid pathway automatically, and it never ran an income test. Missouri's ABD pathway applies an 85% FPL limit. The head's full Social Security plus pension (about $27,510 after the $20 disregard, or 172% of FPL) fails that limit, and at 85 the head is outside the MAGI adult group (ages 19–64)."
+us,scenario_021,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly named the roughly 85% FPL income limit for MO HealthNet ABD and correctly put countable income at about $27,508. It then never compared the two: $27,508 is about 172% of FPL, double the limit. Passing the asset test does not cure an income failure, and an 85-year-old has no MAGI category to fall back on."
+us,scenario_021,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model assumed ABD income disregards and the treatment of medical expenses would bring $27,508 of gross income under the limit. The only disregard is the $20 general unearned-income disregard, and PolicyEngine applies no medical-expense spend-down. Countable income stays near 172% of FPL, far above Missouri's 85% FPL ABD limit."
+us,scenario_021,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model claimed Social Security is 'mostly excluded' from countable income. In fact the ABD test counts all $22,000 as unearned income, which puts the head at about 172% of FPL against Missouri's 85% FPL ABD limit. The model also leaned on Medicare Savings Program pathways (QMB/SLMB/QI). These are not Medicaid eligibility categories, and their limits of 100–135% of FPL are below this income anyway."
+us,scenario_021,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model called the head's income 'very low' without computing it. Social Security of $22,000 plus pension of about $5,530 comes to roughly 172% of FPL, which fails Missouri's 85% FPL aged/blind/disabled income limit. At 85 the head also cannot qualify through the MAGI adult group, which covers ages 19–64, so no pathway is left."
+us,scenario_021,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model looked only at the low taxable income and the modest resources. The ABD income test counts gross Social Security and pension, about $27,510 after the $20 disregard, which is about 172% of FPL and well above Missouri's 85% FPL ABD limit. Age 85 also rules out the MAGI adult expansion category."
us,scenario_021,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly treated $22,000 of Social Security retirement benefits as covered wages subject to payroll tax. It also applied the combined employer-and-employee Social Security and Medicare rates of 12.4% and 2.9%, even though the requested output includes only employee-side taxes; the listed household has no payroll-taxable earnings, so the tax is $0."
us,scenario_021,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_021,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -1471,183 +1530,187 @@ us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model
us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract.
us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits."
us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model filed single and applied the $3,951 senior deduction and the $910 QBI deduction as the engine does. It treated California income tax withheld as unknown and put only the $8,829 of property tax in SALT, and it skipped the 2026 0.5%-of-AGI charitable floor, so it itemized $24,222 and got taxable income of $80,066. The reference adds the engine's $6,058.85 estimate of California income tax withheld, which the prompt does not list, for $29,735.52 itemized and $74,552.59 taxable. With no state income tax paid, the engine's SALT takes its general sales tax estimate, $1,155.60 ($963 state plus a $192.60 local estimate for an unlisted locality), and the output is $12,192.29, or $12,234.66 without the local part; the model's $12,327 also leaves out that sales tax and the charitable floor."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model used qualifying surviving spouse status (joint brackets and the $33,850 standard deduction), but that status requires a dependent child, so single filing applies. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 of MAGI, and it added a non-itemizer charitable deduction, although a single filer here itemizes: $29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $24,832.27 with no state income tax paid, both above the $18,150 single standard deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model included 100% of Social Security instead of the 85% taxable portion. It deducted the $8,351 of unreimbursed employee expenses above the line, although they are not deductible, and it used head-of-household status and brackets instead of single. It never itemized or took the senior and QBI deductions, and its submitted $11,316 does not even match the $10,564 it computed."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the head as a qualifying surviving spouse with MFJ brackets and a $33,950 standard deduction, but with no dependent child the correct status is single, with 22% applying above $50,400. It also omitted the phased $3,951 senior deduction. It put no state income tax in SALT, which the prompt does not list; the reference adds the engine's $6,058.85 estimate of California income tax withheld, for $29,735.52 itemized. As a single filer the head itemizes either way, since the model's own $24,222 of property tax and charity already exceeds the $18,150 single standard deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"The model used qualifying surviving spouse (MFJ) brackets even though no dependent child exists, so single status applies. It imputed $12,771 of mortgage interest from a balance when no interest paid was given, and it took the full $6,000 senior deduction without the phase-down. Its final $6,919 does not match its own $7,350 computation."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $24,222, with no state income tax in SALT and no 0.5%-of-AGI charitable floor; the reference's SALT adds the engine's $6,058.85 estimate of California income tax withheld, which the prompt does not list, for $29,735.52 itemized. It then took an unphased $6,000 senior deduction, which the engine phases down to $3,951.05. On its own $78,017 of taxable income, the single brackets give about $11,876, so its $8,547 is an arithmetic error."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied qualifying surviving spouse joint brackets and the $33,850 standard deduction, but that status requires a dependent child and single status applies. A single filer here itemizes under every reading of the unlisted state income tax: $29,735.52 in the reference (SALT with the engine's $6,058.85 estimate of California income tax withheld, plus charity after the 0.5% floor) and $24,832.27 with no state income tax paid, both above the $18,150 standard deduction. The tax falls in the 22% bracket."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed TCJA expired in 2026, bringing back personal exemptions, 2% miscellaneous deductions and 15%/25% brackets, but OBBBA made the TCJA rate and deduction structure permanent. It also used surviving spouse (MFJ) status without a qualifying child and imputed mortgage interest. Its submitted $13,013 matches none of its own intermediate results (about $5,920–$7,314)."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model used MFJ/surviving spouse treatment with a $32,600 standard deduction, but with no dependent child the head files single. It skipped the senior and QBI deductions, and it held SALT, property tax plus other state taxes, to the old $10,000 cap instead of the 2026 $40,400 cap. Under that cap SALT is $14,887.85 in the reference, which adds the engine's $6,058.85 estimate of California income tax withheld, and $11,334.87 with the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts). Its final $6,800 is also below its own $8,700 estimate."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model filed as surviving spouse with joint brackets and the $33,850 standard deduction, but that status requires a dependent child, so single applies. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 of MAGI (which leaves $3,951.05) and omitted the $910 QBI deduction on rental income."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"The model used surviving spouse joint treatment (a $32,500 standard deduction and the 2025 MFJ $23,850 10% bracket), but the head has no dependent child and files single. It also omitted the phased $3,951 senior deduction and the $910 QBI deduction, and it put no state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld)."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied pre-TCJA law for 2026: a $5,300 personal exemption, 2%-floor unreimbursed employee expenses and 15%/25% brackets. OBBBA made the TCJA brackets permanent, kept miscellaneous itemized deductions suspended, and kept personal exemptions at zero. In SALT it put $2,019 of state income tax, the same figure as its California income tax answer; the engine's own California income tax for the household is $2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld. It also skipped the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"The model used expired pre-TCJA rules: a $5,200 personal exemption, $6,168 of 2%-floor employee expenses and old 15%/25% single brackets, instead of 2026 OBBBA law, which has no exemptions, no miscellaneous itemized deductions, and 10/12/22% brackets. It skipped the senior and QBI deductions, and its submitted $13,047.69 contradicts its own stated $12,372."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized only property tax and charity ($24,222), with no state income tax in SALT and no 0.5% charitable floor; the reference's SALT adds the engine's $6,058.85 estimate of California income tax withheld, which the prompt does not list. It omitted both the $3,951.05 phased senior deduction and the $910 QBI deduction, so it taxed $84,926.75 (the reference taxes $74,552.59), using bracket widths that do not match the 2026 single schedule."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied pre-TCJA 2026 law: a $5,000 personal exemption, a deduction for unreimbursed employee expenses above a 2% floor, and 10%/15%/25% brackets. Under OBBBA, exemptions stay at zero, miscellaneous itemized deductions stay suspended, and the 10/12/22% brackets apply. It also put no state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld), and skipped the $3,951 senior deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model said it itemized mortgage interest, but no interest paid is given, only a loan balance. It never mentioned state income tax in SALT, the 0.5% charitable floor, the $3,951.05 phased senior deduction or the $910 QBI deduction. Its $11,335 matches none of the engine's values: $11,113.57 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, $11,895.23 with the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts) in SALT, and $12,192.29 with no state income tax paid."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model used pre-TCJA itemized deductions (about $33,297, including 2%-floor employee expenses) plus personal exemptions, even though OBBBA permanently suspended both for 2026. It also left out the $3,951.05 senior and $910 QBI deductions, which with the reference's $29,735.52 of itemized deductions bring taxable income to $74,552.59, taxed under the 10/12/22% single brackets."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model computed tax under TCJA-expiration rules: a $5,050 personal exemption, $6,168 of 2%-floor employee expenses and post-sunset rates. OBBBA kept the TCJA brackets, zero exemptions and the suspension of miscellaneous deductions. It put $2,172 of California income tax in SALT, its own estimate; the engine's own California income tax for the household is $2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld. It also omitted the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"The model gave no derivation. Its $13,446 corresponds to about $85,150 of single-filer taxable income, close to AGI minus only property tax and charity ($84,926.75). That suggests it put no state income tax in SALT and left out the $3,951.05 phased senior deduction and the $910 QBI deduction. With those deductions the reference's taxable income is $74,552.59; its SALT adds the engine's $6,058.85 estimate of California income tax withheld, which the prompt does not list."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the old $10,000 SALT cap instead of the 2026 OBBBA cap of $40,400, under which SALT is deductible in full: $14,887.85 in the reference (property tax plus the engine's $6,058.85 estimate of California income tax withheld) and $11,334.87 with the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts). It also skipped the 0.5% charitable floor and omitted the $3,951.05 phased senior deduction, leaving taxable income at $82,846 (the reference's is $74,552.59)."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model's $9,260 is consistent with joint/surviving-spouse brackets (a 12% top bracket) applied to about $81,300 of taxable income. The head has no dependent child and files single, where income above $50,400 is taxed at 22%. It also never mentioned the phased senior deduction or any state income tax in SALT, which the prompt does not list (the reference's SALT adds the engine's $6,058.85 estimate of California income tax withheld)."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized unreimbursed employee business expenses, which are permanently nondeductible under OBBBA. It omitted the $3,951.05 phased senior deduction and the $910 QBI deduction and listed no state income tax in SALT. Its $13,200 is above the engine's value under every reading of that unlisted tax: $11,113.57 in the reference, which adds the engine's $6,058.85 estimate of California income tax withheld, $11,895.23 with the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts) and $12,192.29 with none paid."
-us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"The model filed single but used 2025 brackets and a $15,000 standard deduction. It put no state income tax in SALT and skipped the 0.5% charitable floor, and it omitted the OBBBA senior deduction ($6,000 phased down to $3,951.05). As a result it taxed $84,016.75; the reference taxes $74,552.59, with SALT that adds the engine's $6,058.85 estimate of California income tax withheld, which the prompt does not list."
-us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted $5,850 of federal withholding that does not appear in the facts, and withholding is never subtracted from tax before refundable credits in any case. It also treated the $8,351 of unreimbursed employee expenses as an above-the-line adjustment and added tax-exempt pension income to AGI. It used the standard deduction although the head itemizes under every reading of the unlisted state income tax ($29,735.52 in the reference, $24,832.27 with none paid, against the $18,150 standard deduction), and it omitted the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed deductions and credits wipe out all tax, but AGI is $109,149.16 and the engine's total deductions are at most $34,596.57 under any reading of the unlisted state income tax (the reference's figure, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld). That leaves at least $74,552.59 of taxable income, and the engine allows no nonrefundable credit, the elderly credit included. Its assertion that there is no taxable wage income ignores the $52,635 of wages."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"The model's $1,237 implies only about $12,400 of taxable income, which means it deducted roughly $96,800 from $109,149.16 of AGI. The reference's deductions are $29,735.52 itemized (with SALT that adds the engine's $6,058.85 estimate of California income tax withheld), $910 QBI and $3,951.05 senior, leaving $74,552.59 taxable in the 22% bracket, and the other readings of that unlisted tax leave more. The engine allows no nonrefundable credits."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model used surviving-spouse joint brackets, but with no dependent child the head files single. It also applied a pre-TCJA personal exemption, which OBBBA made permanently zero, and omitted the phased $3,951 senior deduction and the $910 QBI deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Under single rates, the model's $10,247 implies about $70,600 of taxable income, which means it claimed roughly $3,900 more than the reference's $34,596.57 of total deductions. The reference's stack is $29,735.52 itemized (with charity reduced by the 0.5% AGI floor and SALT that adds the engine's $6,058.85 estimate of California income tax withheld), $910 QBI and the senior deduction phased down to $3,951.05, which leaves $74,552.59 taxable. The other readings of that unlisted tax give smaller deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied surviving-spouse joint-rate treatment and a joint-level age-adjusted standard deduction, but that status requires a dependent child. Single status taxes income above $50,400 at 22%, and a single filer here itemizes: $29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $24,832.27 with no state income tax paid, both above the $18,150 single standard deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"The model used the surviving-spouse (joint) tax schedule, but the head has no dependent child and files single. It also never mentioned the $3,951.05 senior deduction or the $910 QBI deduction, which help bring the reference's taxable income to $74,552.59."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"The model's senior deduction ($3,951.08) and QBI deduction ($910) were correct, but it used qualifying surviving spouse status, which requires a dependent child, taking the $33,850 joint standard deduction and joint brackets. As a single filer the head itemizes ($29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld), and taxable income above $50,400 is taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Under single rates, the model's $9,651 implies about $67,900 of taxable income, roughly $6,600 below the reference's $74,552.59, so its itemized, senior and QBI deductions total well above the reference's $34,596.57. The reference itemizes $14,887.85 of SALT, which adds the engine's $6,058.85 estimate of California income tax withheld, plus $14,847.67 of charity left after the 0.5% floor; the other readings of that unlisted tax give less."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model filed single and phased the senior deduction correctly to $3,951, and it itemized about $26,456 without a breakdown. That is $273.46 above the $26,182.54 the engine itemizes with its own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts) in SALT and $3,279.52 below the reference's $29,735.52, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld. It also left out the $910 QBI deduction on rental income, so it taxed about $78,742; with the engine's California income tax in SALT, taxable income is $78,105.58 and the output $11,895.23, against its $12,035."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model used surviving-spouse joint brackets and the $33,850 joint age-adjusted standard deduction, but that status requires a dependent child and single status applies. As a single filer the head itemizes and the 22% rate applies above $50,400, which gives $11,113.57 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $11,895.23 with the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts), instead of $7,956.52."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied a $30,000 QSS standard deduction and QSS brackets, but the head has no dependent child and files single. It also left out itemizing ($29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and at least $24,832.27 under each reading of that unlisted tax), the phased $3,951.05 senior deduction and the $910 QBI deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed TCJA sunset, deducting 2%-floor employee expenses and a $5,419 personal exemption under pre-TCJA single brackets. OBBBA permanently kept those deductions at zero and kept the 10/12/22% schedule. In SALT it put $2,103 of California income tax, the same figure as its California income tax answer; the engine's own California income tax for the household is $2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld. It also omitted the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model's structure matched the engine's: single status, itemizing, a $3,951 senior deduction and a $910 QBI deduction. In SALT it put $1,974 of California income tax, the same figure as its California income tax answer, and it skipped the 0.5% AGI charitable floor, for taxable income of $78,092. With the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts) in SALT, taxable income is $78,105.58 and the output $11,895.23; the reference instead uses the engine's $6,058.85 estimate of California income tax withheld, for $74,552.59 and $11,113.57. It also used bracket amounts that do not match the 2026 single schedule, reaching $11,953."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model put $1,867 of California income tax in SALT, the same figure as its California income tax answer, for SALT of $10,696. With the engine's own California income tax ($2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts) SALT is $11,334.87, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld, for $14,887.85. It dropped the $910 QBI deduction on rental income, skipped the 0.5% charitable floor and used 2026 bracket amounts that are wrong ($12,225/$49,675 instead of $12,400/$50,400). Its $12,192 lands within $1 of the $12,192.29 the engine gives with no state income tax paid, by a different route."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model included 100% of Social Security in AGI instead of the 85% taxable portion, and it applied pre-TCJA law, with 2%-floor employee expenses and 15%/25% brackets, that OBBBA permanently displaced. It also skipped the senior and QBI deductions, and in SALT it put $2,254 of California income tax, the same figure as its California income tax answer; the engine's own California income tax for the household is $2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld."
-us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted about $901 of rental depreciation that is not in the facts, which lowered AGI and threw off the senior-deduction phase-down. In SALT it put about $2,024 of California income tax, the same figure as its California income tax answer; the engine's own California income tax for the household is $2,505.87 on policyengine-us 2.15.17 with California's 2025 amounts, and the reference instead uses the engine's $6,058.85 estimate of California income tax withheld. It also omitted the $910 QBI deduction, and its tax does not follow from the 2026 single brackets on its own $78,538 of taxable income, which give $11,990.36. Its $12,192 lands within $1 of the $12,192.29 the engine gives with no state income tax paid, by a different route."
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"The model's structure matched the engine's (single status, the $40,400 SALT cap, a $3,951.08 senior deduction and the 2026 brackets). In SALT it put $2,700 of California income tax, the same figure as its California income tax answer and $194.13 above the engine's own $2,505.87 (policyengine-us 2.15.17 with California's 2025 amounts); the reference instead uses the engine's $6,058.85 estimate of California income tax withheld. It also skipped the 0.5% AGI charitable floor and declined the $910 QBI deduction on rental income, which gave $78,275.67 of taxable income, against $78,105.58 with the engine's California income tax in SALT and $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed taxable income is too low to owe tax, but AGI is $109,149.16. Across the readings of the unlisted state income tax the engine's deductions are at most the reference's: $29,735.52 itemized, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, plus $910 QBI and a $3,951.05 senior deduction. Those leave $74,552.59 taxable, which puts the head in the 22% bracket."
-us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model filed as surviving spouse with MFJ rates and the $33,850 joint standard deduction, although no dependent child exists and single status applies. It took the full $6,000 senior deduction, wrongly treating $150,000 as the phase-out start instead of $75,000 of MAGI, and it denied the $910 QBI deduction on rental income."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model added the $17,624.75 of taxable Social Security twice, which inflated AGI to $126,773.50 instead of $109,149.16. It also imputed $12,771 of mortgage interest from the loan balance and applied qualifying surviving spouse brackets even though there is no dependent child."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model counted non-deductible items (employee business expenses, health premiums below the 7.5% floor, and invented mortgage interest) as itemized deductions and concluded taxable income was negative. Across the readings of the unlisted state income tax the engine itemizes at most $29,735.52, the reference's, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld; with the $910 QBI and $3,951.05 senior deductions that leaves at least $74,552.59 of taxable income, and the engine allows no nonrefundable credit to offset the tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It got single status, the 2026 single brackets, the $3,951 phased senior deduction and the $910 QBI deduction right. However, it itemized only $8,829 of property tax plus $15,393 of charity. It left out the ~$6,059 of California income tax that belongs in SALT and skipped the 0.5%-of-AGI charitable floor, so it reached taxable income of $80,066 instead of $74,552.59 and overstated tax by about $1,213."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse with joint brackets and the $32,200 joint standard deduction. That status requires a dependent child, so this head files single, where itemized deductions of $29,735.52 beat the $18,150 standard deduction. It also took the full $6,000 senior deduction instead of phasing it down by 6% of MAGI above $75,000 to $3,951."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used head-of-household status, which needs a qualifying person. It also subtracted the $8,351 of unreimbursed employee expenses above the line, though they are not deductible at all, and counted 100% of Social Security instead of the 85% maximum. Its reported $11,316 does not match its own $10,564 computation; correct single filing with $34,596.57 of total deductions gives $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed taxable Social Security was about $0 and that the senior standard deduction wiped out taxable income. In fact, $91,524 of other income pushes provisional income far over the thresholds, so 85% of benefits ($17,624.75) is taxable. AGI is $109,149.16, and after $34,596.57 of deductions, $74,552.59 is taxable, producing $11,113.57 rather than $0."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse joint brackets and a $33,950 standard deduction, but without a dependent child the head files single. It also left out the OBBBA senior deduction ($3,951 after phase-down) and the California income tax in SALT. Under single status, $29,735.52 of itemized deductions plus QBI and the senior deduction give taxable income of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse with joint brackets despite having no dependent child. It also made up ~$12,771 of mortgage interest from the loan balance when no interest paid is listed, and took the unphased $6,000 senior deduction instead of $3,951. Its reported $6,919 does not follow from its own ~$7,350 estimate."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It took the full $6,000 senior deduction, but MAGI of $109,149 cuts that to $3,951. It also left the California income tax out of SALT and skipped the 0.5% charitable floor. It then reported $8,547 on ~$78,017 of taxable income, which single brackets cannot produce (they give about $12,100)."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied qualifying-surviving-spouse joint brackets and the $33,850 joint standard deduction, but that status requires a dependent child, so the head files single. As a single filer, itemized deductions of $29,735.52 beat the $18,150 standard deduction. With the $910 QBI and $3,951 senior deductions, $74,552.59 is taxed at 10%/12%/22% for $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired for 2026 and brought back personal exemptions, the 2%-floor deduction for employee expenses, and 15%/25% brackets, although OBBBA made the TCJA rules permanent. It also made up $13,835 of mortgage interest and used joint surviving-spouse rates without a dependent child. Its submitted $13,013 matches none of its own intermediate results."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse status with a $32,600 joint standard deduction although it requires a dependent child. It left out the senior deduction, the QBI deduction, and the California income tax in SALT. It then reported $6,800, which contradicts its own ~$8,700 estimate; single status with $34,596.57 of deductions gives $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse on joint brackets with the $33,850 joint standard deduction, but without a dependent child the head files single and itemizes $29,735.52. It also took the full $6,000 senior deduction instead of phasing it down by 6% of MAGI over $75,000 to $3,951, and left out the $910 QBI deduction on rental income."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse status with a $32,500 joint standard deduction and joint 10%/12% brackets, but the head has no dependent child and files single. It also skipped the $3,951 phased senior deduction and the $910 QBI deduction, and left California income tax out of SALT."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026: a $5,300 personal exemption, a 2%-floor deduction for $8,351 of employee expenses, and 15%/25% brackets. OBBBA made the TCJA 10/12/22% schedule and the repeal of those items permanent. It also understated California income tax in SALT at $2,019 (the correct figure is ~$6,059) and left out the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026: a $5,200 personal exemption, a 2%-floor deduction for employee expenses, and 15%/25% single brackets, although OBBBA made the TCJA rules permanent. It also understated California income tax in SALT and left out the senior and QBI deductions. Its submitted $13,047.69 does not even match the $12,372 in its own explanation."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status but itemized only $24,222 of property tax and charity, leaving out ~$6,059 of California income tax in SALT. It never subtracted the $3,951 phased senior deduction or the $910 QBI deduction, so taxable income was $84,927 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA rules: a $5,000 personal exemption, employee expenses deducted above a 2% floor, and 15%/25% single brackets. OBBBA made the TCJA 10/12/22% schedule permanent and repealed those deductions for good. It also left out the $3,951 phased senior deduction and the California income tax in SALT."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized mortgage interest that is not an input; only the loan balance is listed, so interest is zero. It never accounts for the ~$6,059 California income tax in SALT, the 0.5% charitable floor, the $3,951 senior deduction, or the $910 QBI deduction that together bring taxable income to $74,552.59. Its $11,335 corresponds to about $1,000 less in deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA itemized deductions (~$33,297, including miscellaneous employee expenses), personal exemptions, and pre-TCJA single brackets. OBBBA made the TCJA rules permanent, so 2026 has no personal exemption and no miscellaneous itemized deduction, and uses the 10/12/22% schedule. It also left out the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a $5,050 personal exemption and $6,168 of employee expenses above a 2% floor, and used post-sunset bracket rates; OBBBA permanently eliminated these and kept the 10/12/22% schedule. It also understated California income tax in SALT at $2,172 instead of ~$6,059 and left out the senior and QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. At 2026 single rates, $13,446 corresponds to about $85,150 of taxable income, which is AGI minus only ~$24,000 of property tax and charity. That means it left out the ~$6,059 California income tax in SALT, the $3,951 phased senior deduction, and the $910 QBI deduction, which bring taxable income to $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 limit, but OBBBA raised the 2026 cap to $40,400, so the full $14,887.85 of property tax plus California income tax is deductible. It also left out the $3,951 phased senior deduction for the 77-year-old, so taxable income stayed at $82,846 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $9,260 matches joint-rate (surviving-spouse) tax on about $81,000 of taxable income, after itemizing only property tax and charity. The head has no dependent child and files single, where income above $50,400 is taxed at 22%. Correct single-status deductions of $34,596.57 produce $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $13,200 matches single-rate tax on about $84,000 of taxable income, which is AGI minus only property tax, charity, and QBI. It left out the ~$6,059 California income tax in SALT and the $3,951 phased senior deduction. Its reference to deducting unreimbursed employee expenses is also wrong, because OBBBA permanently eliminated miscellaneous itemized deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2025 single brackets and the 2025 standard deduction instead of 2026 values. It left out the OBBBA senior deduction ($3,951 after phase-down), and it itemized only $24,222, without the ~$6,059 California income tax in SALT or the 0.5% charitable floor. As a result, taxable income was $84,016.75 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $8,351 of unreimbursed employee expenses as an above-the-line deduction, though they are not deductible, and added the $371 tax-exempt pension to AGI. It then subtracted $5,850 of federal withholding that is not an input and is not a nonrefundable credit. Its own arithmetic is also wrong: $84,169 of single taxable income gives about $13,200 of tax, not $7,102."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed no wage income remained taxable after pension and Social Security offsets and that deductions wiped out the tax. In fact, $52,635 of wages, $30,989.41 of pension, and 85% of Social Security are all taxable, giving AGI of $109,149.16. Total deductions are only $34,596.57, leaving $74,552.59 taxable and $11,113.57 of tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It offered only a generic description, and its $1,237 implies roughly $12,000 of taxable income. AGI of $109,149.16 minus $34,596.57 of deductions ($29,735.52 itemized, $910 QBI, $3,951.05 senior) leaves $74,552.59 taxable, which is $11,113.57 at 2026 single rates."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets although the head has no dependent child and files single. It also subtracted a personal exemption that OBBBA permanently set to zero. Its ~$30,390 of itemized deductions came from counting employee expenses, which are no longer deductible, rather than the California income tax in SALT."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"At single rates, its $10,247 corresponds to about $70,600 of taxable income, so it took about $3,940 more in deductions than the correct $34,596.57. The senior deduction is only $3,951 after the 6% phase-down above $75,000 MAGI, charity is reduced by the 0.5%-of-AGI floor, and the only other allowable itemized item is $14,887.85 of SALT."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint rates and the joint age-based standard deduction, but that status requires a dependent child. The head files single, itemizes $29,735.52 (more than the $18,150 single senior standard deduction), and owes $11,113.57 at single rates."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) schedule, but without a dependent child the head files single, where income above $50,400 is taxed at 22%. Its explanation also names no senior or QBI deduction, and it left the California income tax out of the itemized SALT."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, $3,951 senior deduction and $910 QBI deduction all match the correct values. However, it applied surviving-spouse status with joint brackets and the $33,850 joint standard deduction, which requires a dependent child. The head files single, itemizes $29,735.52 instead of taking the $18,150 standard deduction, and owes $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"At single rates, its $9,651 corresponds to about $67,900 of taxable income, so it took about $6,650 more in deductions than the correct $34,596.57. Itemized deductions are limited to $14,887.85 of SALT and charity net of the 0.5% AGI floor, with no mortgage interest and no employee expenses, alongside $910 of QBI and the $3,951 senior deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status and the $3,951 phased senior deduction correctly, but itemized only $26,456, counting just ~$2,234 of California income tax instead of ~$6,059 and skipping the 0.5% charitable floor. It also left out the $910 QBI deduction on rental income, so taxable income was about $78,742 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, $3,951 senior deduction and $910 QBI deduction match the correct values, but it applied surviving-spouse joint brackets and the $33,850 joint standard deduction. That status requires a dependent child, so the head files single, itemizes $29,735.52, and owes $11,113.57 at single rates."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets and a $30,000 standard deduction although that status requires a dependent child. It also left out the $3,951 phased senior deduction, the $910 QBI deduction, and the $29,735.52 of itemized deductions that apply under single status."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law: a $5,419 personal exemption, $6,168 of employee expenses above a 2% floor, and 15%/25% single brackets. OBBBA made the TCJA rules permanent, so none of these apply in 2026. It also understated California income tax in SALT and left out the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status, the phased $3,951 senior deduction and the $910 QBI deduction correctly. However, it put only $1,974 of California income tax into SALT instead of ~$6,059, and it skipped the 0.5%-of-AGI charitable floor. As a result, it itemized $26,196 instead of $29,735.52 and reached taxable income of $78,092 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $1,867 of California income tax in SALT instead of ~$6,059 and set the QBI deduction to $0, although rental income gives a $910 deduction. It also used estimated brackets (10% to $12,225, 12% to $49,675) instead of the 2026 single thresholds of $12,400 and $50,400."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It included 100% of Social Security ($20,735) in AGI instead of the 85% maximum ($17,624.75). It also applied post-sunset 15%/25% brackets and a 2%-floor deduction for employee expenses, though OBBBA made the TCJA rules permanent. On top of that, it understated California income tax in SALT and left out the senior and QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted ~$901 of rental depreciation that is not an input, and it counted only ~$2,024 of California income tax in SALT instead of ~$6,059. It also left out the $910 QBI deduction on rental income, so taxable income was $78,538 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $11,113.57 single-filer liability."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status and the $3,951.08 phased senior deduction correctly, but deliberately took no QBI deduction, although 20% of the $4,550 of rental income ($910) applies. It also estimated California income tax at $2,700 instead of ~$6,059 and skipped the 0.5% charitable floor, so taxable income was $78,275.67 instead of $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted that the standard deduction left too little taxable income for any tax. In fact AGI is $109,149.16 and total deductions are only $34,596.57, so $74,552.59 is taxable, producing $11,113.57 of tax rather than $0."
+us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse using joint brackets, the $33,850 joint standard deduction, and the full $6,000 senior deduction. Without a dependent child the head files single. The senior deduction phases down by 6% of MAGI above $75,000 to $3,951, and the $910 QBI deduction on rental income applies."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It computed AGI as $126,773.50 by adding taxable Social Security on top of provisional income, which already includes half of the benefits; correct AGI is $109,149.16. It also made up $12,771 of mortgage interest from the loan balance when no interest is listed, and applied surviving-spouse joint brackets despite having no dependent child."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed itemized deductions produced negative taxable income, counting health premiums that fall below the 7.5% floor, nondeductible employee expenses, and made-up mortgage interest. Allowable itemized deductions are only $29,735.52, leaving $74,552.59 taxable and $11,113.57 of tax."
us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies."
us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). It then submitted $4,557.61, which does not match its own stated sum of $4,658.20 or any other figure in its reasoning."
-us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal OASDI and Medicare ($4,026.58). It left out California's mandatory employee SDI contribution of 1.3% x $52,635 = $684.26, which the output definition explicitly includes as a mandatory employee state payroll tax."
-us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax, so it omitted the $684.26 employee SDI contribution (1.3% of wages). It also miscomputed Medicare as $762.71 instead of $763.21 and submitted $4,028.27, which does not match its own stated total of $4,026.08."
-us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly included uncapped CA SDI but used a 1.2% rate ($631.62) instead of the 2026 employee SDI rate of 1.3% ($684.26), which leaves the total $52.64 short."
-us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Recognized CA SDI but excluded it on the claim that the instructions cover only Social Security and Medicare. The output definition explicitly includes mandatory employee state payroll taxes, so the 1.3% SDI contribution of $684.26 belongs in the total."
-us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and omitted the 1.3% CA SDI contribution ($684.26). It then truncated its own FICA total of $4,026.58 to $4,026."
-us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Included uncapped CA SDI but applied the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26), which leaves the total $105.27 short."
-us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said CA SDI is not a mandatory employee tax and omitted the 1.3% SDI contribution ($684.26). It then submitted $3,264.37, which is roughly the Social Security piece alone and contradicts its own FICA total of $4,026.58."
-us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Stated that no state payroll tax applies, so it omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also slightly miscomputed Medicare as $763.24."
-us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"Included CA SDI but at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26). It also rounded each component to whole dollars, reaching $4,605."
-us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 employee rate of 1.3% ($684.26), which leaves the total $52.63 short."
-us,scenario_022,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 employee rate of 1.3% ($684.26), which leaves the total $52.63 short."
-us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,Correctly treated CA SDI as uncapped but used the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26).
-us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Summed only Social Security and Medicare, then rounded to $4,027. It omitted California's mandatory 1.3% employee SDI contribution ($684.26), which the output definition includes."
-us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Included CA SDI at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26), and rounded the components to whole dollars."
-us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Included CA SDI at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26), and rounded the components to whole dollars."
-us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare ($4,027). It omitted California's mandatory 1.3% employee SDI contribution of $684.26."
-us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Included CA SDI but at the 2023 rate of 0.9% ($473.72) instead of the 2026 rate of 1.3% ($684.26), which leaves the total $210.53 short."
-us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $4,500.29 equals FICA of $4,026.58 plus CA SDI at the 2023 rate of 0.9% (about $473.71). It needed the 2026 SDI rate of 1.3%, which gives $684.26."
-us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare ($4,026.58). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)."
-us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,Correctly treated CA SDI as a mandatory state payroll tax but used 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26).
-us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Counted only Social Security and Medicare and omitted California's mandatory 1.3% employee SDI contribution ($684.26). It then rounded to $4,026 by citing a whole-dollar rounding rule that does not apply to payroll tax."
-us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Stated that California has no employee state payroll tax, which omits the mandatory 1.3% SDI contribution ($684.26). Its $4,030 also misstates the FICA sum, which is $4,026.58."
-us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also mis-added its own FICA components: 6.2% plus 1.45% of $52,635 is $4,026.58, not $3,308."
-us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% instead of the 2026 employee rate of 1.3%, which undercounts SDI by $52.63 ($631.62 vs. $684.26)."
-us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied only the 7.65% combined FICA rate ($4,027 rounded). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)."
-us,scenario_022,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% instead of the 2026 employee rate of 1.3%, and rounded the total to $4,658 instead of $4,710.83."
-us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare ($4,027). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)."
-us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars."
-us,scenario_022,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Correctly included mandatory CA SDI but at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26), then rounded to $4,658."
-us,scenario_022,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,Correctly included mandatory CA SDI but at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26).
-us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"Used a projected CA SDI rate of 1.2% ($632) instead of the 2026 rate of 1.3% ($684.26). It also assumed a wage base, although CA SDI has had no wage cap since 2024."
-us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,"Approximated CA SDI at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars."
-us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for payroll_tax, so there is no substantive computation to score against $4,710.83."
-us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,Correctly treated CA SDI as mandatory and uncapped but applied 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26).
-us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Summed only Social Security and Medicare and omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also truncated $4,026.58 to $4,026."
-us,scenario_022,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"Included mandatory CA SDI but at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars."
-us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"Applied CA SDI at the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26). It also cited a wage cap near $153,164, although CA SDI has no wage cap."
-us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Stated that no mandatory employee state payroll taxes apply, which omits California's 1.3% employee SDI contribution of $684.26."
+us,scenario_022,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used California SDI at 1.2%, which is the 2025 rate, instead of the 2026 rate of 1.3% ($684.26). It then submitted $4,557.61, which does not match even its own $4,658.20 total."
+us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal Social Security and Medicare ($4,026.58). It left out California's mandatory employee SDI contribution of 1.3% of wages ($684.26), which is part of the requested payroll tax."
+us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said wrongly that no mandatory California payroll tax applies, so it dropped the 1.3% SDI ($684.26). It also got Medicare wrong ($762.71 instead of $763.21) and then submitted $4,028.27, which does not match its own sum."
+us,scenario_022,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Submitted $0 even though its own explanation gives $4,026 of Social Security and Medicare tax. That $4,026 also leaves out the 1.3% California SDI ($684.26), so the right total is $4,710.83."
+us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Used 1.2% for California SDI ($631.62), which is the 2025 rate. The 2026 rate is 1.3% ($684.26), so the total came out $52.63 short."
+us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Noticed that SDI applies but left it out, wrongly reading the instructions as covering only Social Security and Medicare. The question says to include mandatory employee state payroll taxes, which here is California SDI at 1.3% ($684.26)."
+us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Said wrongly that California has no mandatory employee payroll tax, so it left out the 1.3% SDI contribution ($684.26). It also rounded its federal-only total down to $4,026."
+us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,Used the 2024 California SDI rate of 1.1% ($578.99). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said wrongly that California SDI is voluntary or not an employee tax, so it left out the mandatory 1.3% employee contribution ($684.26). It then submitted $3,264.37, which is roughly the Social Security figure alone and does not match its own $4,026.58 FICA total."
+us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Explicitly left out state payroll tax, so it missed California's mandatory 1.3% SDI ($684.26). It also got Medicare wrong ($763.24 instead of $763.21)."
+us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26). It also rounded each piece to whole dollars.
+us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Correctly noted that SDI has no wage cap, but used the 2024 rate of 1.1% ($578.99). The 2026 rate is 1.3% ($684.26)."
+us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Added only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26).
+us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,Included only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26).
+us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,Used the 2023 California SDI rate of 0.9% ($473.72). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,500.29 total implies California SDI at 0.9% (about $473.71), the 2023 rate. The 2026 rate is 1.3% ($684.26)."
+us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Added only federal Social Security and Medicare ($4,026.58) and left out California's mandatory 1.3% employee SDI ($684.26)."
+us,scenario_022,payroll_tax,glm-5.2,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Left out California's mandatory 1.3% employee SDI ($684.26). It also rounded its federal-only figure down to $4,026, citing a whole-dollar rounding rule that does not exist."
+us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said wrongly that California has no employee payroll tax, so it dropped the 1.3% SDI ($684.26). Its $4,030 is also not the correct federal-only sum of $4,026.58."
+us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out California's 1.3% SDI ($684.26). It also got the federal arithmetic wrong: 7.65% of $52,635 is $4,026.58, not $3,308."
+us,scenario_022,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,Estimated California SDI at the 2025 rate of 1.2%. The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Applied only the 7.65% combined federal FICA rate and left out California's mandatory 1.3% employee SDI ($684.26).
+us,scenario_022,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,Estimated California SDI at the 2025 rate of 1.2%. The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Calculated only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26).
+us,scenario_022,payroll_tax,grok-4.5,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($632). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,grok-4.6,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,grok-4.7,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Used a projected 1.2% California SDI rate ($632), which is the 2025 rate. The 2026 rate is 1.3% ($684.26)."
+us,scenario_022,payroll_tax,inkling,llm_error,state_local_rule,False,Used about 1.2% for California SDI ($632). The 2026 employee rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer or explanation for payroll_tax, so there was no value to score."
+us,scenario_022,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Correctly applied SDI with no wage cap, but used the 2025 rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26)."
+us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Included only Social Security and Medicare, rounded to $4,026, and left out California's mandatory 1.3% employee SDI ($684.26)."
+us,scenario_022,payroll_tax,ox-alpha,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($632). The 2026 rate is 1.3% ($684.26).
+us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Used an outdated 1.1% SDI rate with a wage cap of about $153,164 that no longer exists. In 2026, SDI is 1.3% of all wages ($684.26)."
+us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Said wrongly that no mandatory state payroll taxes apply, so it left out California's 1.3% employee SDI ($684.26)."
us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It filed as a qualifying surviving spouse with joint brackets, but there is no dependent child, so the correct status is single. It also deducted $6,521 of employee business expenses, which PolicyEngine does not allow in California itemized deductions. Its own joint-bracket arithmetic reached about $980, and it then reported an unsupported $3,162 instead of the single-schedule result: $2,752.58 minus $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It used the joint brackets and took two personal credits plus a senior credit ($459). This filer has no dependent, so the correct status is single. The single schedule on $67,847.75 of taxable income gives $2,752.58, less $312.93 of credits. It also deducted the full $15,393 of charitable gifts instead of $14,847.67."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It built taxable income of $106,979 from a made-up federal-style standard deduction and never excluded Social Security or itemized. California taxable income is $91,524.41 AGI minus $23,676.67 of itemized deductions, or $67,847.75. It also subtracted an invented $5,320 Senior Property Tax Postponement 'credit'. That program defers property tax; it is not a nonrefundable income tax credit."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as married filing jointly, using joint brackets and joint-plus-senior credits of about $420. With no dependent child, the filer is single. Single-bracket tax on $67,847.75 is $2,752.58, and the correct credits are $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used joint brackets for a surviving spouse with no dependent, who is actually a single filer. It also made up about $7,000 of mortgage interest that is not in the facts. Its reported $1,856 does not match its own arithmetic: a $1,228 pre-credit tax minus credits cannot produce $1,856."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"Its single-filer setup and $24,222 of itemized deductions were close to correct. However, it put tax on $67,302 at $2,393, while the 2026 single brackets produce about $2,709 at that income. It also missed the reduction of the charitable deduction to $14,847.67, which puts taxable income at $67,847.75, gross tax at $2,752.58, and credits at $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It applied joint brackets and two personal credits plus a senior credit (about $471). The surviving spouse has no dependent child, so the filer is single. The correct result is single-bracket tax of $2,752.58 less one personal and one senior credit ($312.93)."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It used married-filing-jointly brackets, with the 2% bracket running to $49,368, and a joint $444 personal credit plus a senior credit. With no dependent child, the correct status is single. The single schedule on $67,847.75 of taxable income gives $2,752.58 before credits of $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used qualifying-surviving-spouse brackets, but no dependent child is present, so the filer is single. It guessed a flat 2.8% effective rate instead of running the brackets. It also claimed the senior exemption credit was built into the tables, so it never subtracted the $312.93 of personal and senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It applied joint/qualifying-surviving-spouse brackets and got about $1,426 of tax on $67,300. The filer has no dependent and is single, and single-bracket tax on $67,847.75 is $2,752.58. The correct result subtracts $312.93 of personal and senior credits from that figure."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It used federal AGI of $109,149 as California AGI and never subtracted the $17,624.75 of taxable Social Security that California excludes. As a result, taxable income was $84,927 instead of $67,847.75, pushing tax into the 9.3% bracket."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It deducted $6,168 of unreimbursed employee business expenses, which are not part of California itemized deductions here. That lowered taxable income to $61,134 instead of $67,847.75. It also subtracted only one $167 personal credit and omitted the senior exemption credit; the correct credits total $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It included unreimbursed employee business expenses and claimed $30,390 of itemized deductions, but the correct total is $23,676.67 (charity plus real estate taxes). That understated taxable income at $61,134. It applied only one $153 exemption credit, omitting the senior credit. Its submitted $1,964.52 also contradicts its own stated $2,135."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It deducted the full $15,393 of charitable gifts instead of the allowed $14,847.67, so its taxable income was $67,302 instead of $67,847.75. Its bracket arithmetic then overstated tax at $2,780.91, when the correct figure on the higher base is $2,752.58. It also subtracted $298 of credits instead of $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It claimed $30,742.52 of itemized deductions by including $6,521 of employee business expenses above a 2% floor. California itemized deductions here are only charity ($14,847.67) and real estate taxes ($8,829), for $23,676.67. It therefore understated taxable income at $60,781.48 instead of $67,847.75."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It used a standard deduction, even though $23,676.67 of itemized deductions far exceeds the $5,706 single standard deduction. Its $1,175 is also far below the single-bracket tax at any taxable income consistent with the facts. The correct path is $67,847.75 of taxable income, $2,752.58 of tax, and $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It included about $6,521 of unreimbursed employee business expenses and claimed $30,743 of itemized deductions instead of $23,676.67. That understated taxable income by about $7,000, so its single-bracket tax came out below the correct $2,752.58."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It deducted employee business expenses and reached $30,743 of itemized deductions, for $60,781 of taxable income. The correct figures are $23,676.67 and $67,847.75. Its bracket tax of $2,415 on the understated base, minus $288 of credits (not $312.93), produced the wrong total."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no derivation. Its $3,211 exceeds even the correct $2,752.58 gross tax on $67,847.75 of taxable income, which means it both undercounted deductions and failed to subtract the $312.93 of personal and senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"Its $30,743 of itemized deductions explicitly includes employee business expenses. California itemized deductions here are charity plus real estate taxes, $23,676.67. That understated taxable income by about $7,000, and at the 8% marginal rate that is roughly $400 too little tax."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"The correct result is $2,752.58 of single-bracket tax on $67,847.75, less $312.93 of personal and senior credits. Its $2,652 is consistent with gross tax near $2,752 but only about $100 of credits instead of the full $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"Its $2,050 matches the result of deducting about $7,000 of employee business expenses on top of charity and property tax. That understates taxable income below the correct $67,847.75, which is AGI of $91,524.41 minus $23,676.67 of itemized deductions."
-us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It invented $24,556 of rental income, when the facts list $4,550, and a mortgage-interest figure. It also used a federal standard deduction instead of California itemized deductions. Its $11,626 comes from a fabricated income base, not California AGI of $91,524.41 minus $23,676.67."
-us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It added a nonexistent $1,750 pension adjustment, never subtracted the $17,624.75 of taxable Social Security, and used a $7,089 standard deduction even though itemized deductions of $23,676.67 are larger. It then said the senior exemption credit does not apply, dropping the $312.93 of personal and senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed deductions and age-related provisions eliminate California tax. In fact, $91,524.41 of California AGI minus $23,676.67 of itemized deductions leaves $67,847.75 of taxable income, taxed at $2,752.58. Only $312.93 of credits offset that."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It gave no specific derivation. Its $1,036 is consistent with joint/surviving-spouse brackets and joint credits, not the single schedule that applies to a surviving spouse with no dependent. The single schedule gives $2,752.58 on $67,847.75, less $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used joint/surviving-spouse brackets ($1,136 of tax) with joint-plus-senior credits of $486, but a surviving spouse with no dependent files single. It also deducted employee business expenses and reached $60,781 of taxable income instead of $67,847.75."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It subtracted only the personal exemption credit and omitted the senior exemption credit that a 77-year-old filer receives, when the combined credit is $312.93. Its $2,893 exceeds the $2,752.58 gross tax on the correct $67,847.75 base, so it also overstated the bracket tax."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It explicitly applied qualifying-surviving-spouse rates, but that status requires a dependent child, so the filer is single. It also deducted employee business expenses, which are not in the correct $23,676.67 of itemized deductions."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"It applied the surviving-spouse (joint) tax schedule. With no dependent child, the filer is single, and the single schedule on $67,847.75 gives $2,752.58 before $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It used the surviving-spouse schedule and a doubled $306 personal credit plus a $153 senior credit. With no dependent, the filer is single and gets $312.93 in total credits. It also deducted employee business expenses, reaching $30,742.52 of itemized deductions instead of $23,676.67."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It deducted allowable employee expenses in California itemized deductions, but only charity and real estate taxes count ($23,676.67). That understated taxable income below $67,847.75. It also mentions only the personal exemption credit and omits the senior credit."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It deducted employee business expenses and reduced taxable income to about $60,800. The correct taxable income is $67,847.75 because itemized deductions are only $23,676.67 of charity and property tax. It also subtracted only the personal exemption credit, not the combined $312.93 personal and senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It used surviving-spouse brackets and $474 of joint-plus-senior credits, but with no dependent child the filer is single and the credits are $312.93. It also included employee expenses, reaching $30,742.52 of itemized deductions instead of $23,676.67."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It declined to compute and defaulted to $0. The facts yield $67,847.75 of California taxable income ($91,524.41 AGI minus $23,676.67 of itemized deductions), taxed at $2,752.58 on the single schedule before $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It included employee business expenses and reached $30,743 of itemized deductions and $60,781 of taxable income, instead of $23,676.67 and $67,847.75. It also subtracted only the $153 personal credit and omitted the senior exemption credit that makes the total $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It added $6,168 of unreimbursed employee expenses to itemized deductions, reaching $30,390 and $61,134 of taxable income. The correct figures are $23,676.67 of itemized deductions and $67,847.75 of taxable income, with no employee-expense deduction."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It deducted $6,521 of employee expenses above a 2% floor and used the full $15,393 of charity, reaching $30,743 of itemized deductions. The correct deductions are $23,676.67: charity of $14,847.67 plus real estate taxes of $8,829. Its taxable income of $60,781 is therefore about $7,066 too low."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It deducted $6,521 of miscellaneous employee expenses, understating taxable income at $60,781 instead of $67,847.75. It also stated that no nonrefundable credits apply, omitting the $312.93 of personal and senior exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It included miscellaneous employee expenses above the 2% floor, reaching about $30,761 of itemized deductions instead of $23,676.67. It also misstated California AGI as $90,623 instead of $91,524.41. Its taxable income of $59,862 is about $8,000 too low, and it subtracted only the personal credit without the senior credit."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It subtracted only a $144 personal exemption credit and omitted the senior exemption credit a 77-year-old receives; together they total $312.93. It also used the full $15,393 of charity, giving $67,302 of taxable income, and overstated the bracket tax at $2,844."
-us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It asserted income falls below the tax threshold after deductions. California AGI of $91,524.41 minus $23,676.67 of itemized deductions leaves $67,847.75 of taxable income and $2,752.58 of tax before $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It wrongly said California taxes federally taxable Social Security, so it never subtracted $17,624.75 and used $84,927 of taxable income. It also applied joint brackets and two personal credits, when a surviving spouse with no dependent files single."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It used a $10,726 standard deduction instead of itemizing, when the $23,676.67 of charity plus real estate taxes is larger. It then invented a rate schedule starting at 4% and subtracted $639 of credits. The correct path is $67,847.75 of taxable income on the single brackets, less $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It claimed deductions and senior credits reduce tax to zero. Itemized deductions of $23,676.67 leave $67,847.75 of taxable income and $2,752.58 of tax, and senior and personal credits remove only $312.93 of that."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It applied joint/surviving-spouse brackets and joint exemption credits, but a head with no dependent files single. It also took the 2% employee-expense floor from CA AGI ($6,521 instead of $6,167.56). Then it dropped its own ~$980 result and reported an unsupported $3,162, when single-schedule tax on $61,134.44 less $306 of credits is $1,968.81."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It used the joint schedule and two personal credits plus a senior credit ($459), but with no dependent the head files single, so the credits are $306. It also left out the $6,167.56 deduction for unreimbursed employee business expenses (above the 2%-of-federal-AGI floor), which put taxable income at $67,302 instead of $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from a made-up federal taxable income of $134,529 and never removed Social Security to reach the $91,524 CA AGI. It used a standard deduction instead of the $30,390 of CA itemized deductions, then subtracted a fictitious $5,320 'Senior Property Tax Postponement' credit, which is not an income tax credit."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed CA taxable income was near zero. In fact CA AGI after removing Social Security is $91,524 and taxable income after $30,390 of itemized deductions is $61,134.44, which produces $2,274.81 of tax less $306 of exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse and used MFJ brackets on about $66k. With no dependent child the single schedule applies, which gives $2,274.81 on $61,134.44. It also left out the $6,167.56 employee-expense miscellaneous deduction."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used MFJ/surviving-spouse brackets and two personal credits, though the head has no dependent and files single. It also imputed about $7,000 of mortgage interest that was never given. Its own arithmetic ($1,228 less credits) does not support the $1,856 it reported."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out California's miscellaneous itemized deduction for unreimbursed employee business expenses ($8,351 less 2% of federal AGI = $6,167.56), so taxable income came to $67,302 instead of $61,134.44. Its single-bracket tax of $2,393 is also wrong for that income level."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It used joint brackets and two personal credits plus one senior credit ($471), but a surviving spouse with no dependent child files single and gets $306 of credits. It also left out the $6,167.56 employee-expense miscellaneous deduction, so taxable income was $67,302 instead of $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly concluded that California conforms to the TCJA suspension of unreimbursed employee expenses. California still allows them above a 2%-of-federal-AGI floor ($6,167.56). It also used MFJ brackets and $588 of credits, when the no-dependent surviving spouse files single and gets $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used qualifying-widow(er) brackets, though the head has no dependent and files single. It left out the $6,167.56 employee-expense deduction. It also skipped the $306 personal-plus-senior exemption credits, wrongly saying they were 'embedded in bracket tables'."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used joint/QSS brackets on $67,300 of taxable income, but the single schedule applies because there is no qualifying dependent. It also left out the $6,167.56 deduction for unreimbursed employee business expenses that California allows above the 2% floor."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $109,149 without subtracting the $17,624.75 of taxable Social Security that California excludes. It also left out the $6,167.56 employee-expense deduction, which inflated taxable income to $84,927 instead of $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It reached the correct $61,134.44 of taxable income. However, it understated single-schedule tax ($2,186 instead of $2,274.81) and subtracted only a $167 personal credit, missing the separate $153 senior exemption credit for a filer aged 65+ (total $306)."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its explanation computes $2,288 less a single $153 credit, which leaves out the $153 senior exemption credit. The submitted $1,964.52 contradicts that explanation and implies bracket tax of $2,270.52 instead of $2,274.81 on $61,134.44, before the $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It counted only charitable gifts and property tax ($24,222) and left out California's miscellaneous itemized deduction for unreimbursed employee business expenses above 2% of federal AGI ($6,167.56). That taxed $67,302 instead of $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It took the 2% employee-expense floor from CA AGI ($91,524) instead of federal AGI ($109,149), overstating the deduction by about $353 ($6,521 instead of $6,167.56). It also took $310 of exemption credits instead of $306, which put the result slightly below the correct figure."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It said it used a single 65+ standard deduction, but itemized deductions of $30,389.97 far exceed the $5,706 CA standard deduction. Its $1,175 does not come from any bracket computation; the correct path is $2,274.81 of tax on $61,134.44 less $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI (itemized $30,743 instead of $30,389.97). Its $2,180 is also higher than single-schedule tax on its own $60,781 of taxable income less the $306 personal and senior credits, so it misapplied the brackets or the credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI instead of federal AGI and taxed $60,781. It then overstated the single-schedule tax ($2,415, when the 2026 brackets give about $2,247 on that income) and took $288 of credits instead of $306."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It showed no derivation and reported $3,211, about $1,242 too high. That gap equals roughly $15k of extra income at the 8% bracket, so it did not tax only the $61,134.44 left after excluding Social Security and taking the $30,390 of itemized deductions, before subtracting the $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI (itemized $30,743 instead of $30,389.97). Its $2,035 also exceeds single-schedule tax on its own $60,781 less $306 of personal and senior credits, so it overstated bracket tax or under-applied the credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its $2,652 is $683 too high, which is about $8.5k of extra income taxed at 8%. That matches leaving out the $6,167.56 employee-expense miscellaneous deduction and under-applying the $306 exemption credits instead of taxing $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It reported $2,050 with no computation, $81 more than single-schedule tax of $2,274.81 on $61,134.44 less $306 of credits. That overstatement equals about $1,000 of extra income at 8%, which matches taking the 2% floor from the wrong AGI or misapplying the brackets."
+us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It made up $24,556 of rental income and a mortgage-interest figure, kept taxable Social Security in the base, and used a standard deduction even though itemized deductions of $30,390 are far larger. That produced an $11,626 tax that bears no relation to the $61,134.44 of CA taxable income."
+us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added a nonexistent tax-exempt pension adjustment to CA AGI and used a standard deduction with a made-up $1,549 senior add-on. It wrongly said that beat itemized deductions, which are actually $30,390. It also denied the $153 senior exemption credit that applies at age 77."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It said age and deductions wipe out all CA tax. But CA taxable income after excluding Social Security and itemizing $30,390 is still $61,134.44, which produces $2,274.81 of tax, and the exemption credits offset only $306 of it."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave a generic explanation and reported $1,036, about $933 too low. That is consistent with using joint/QSS brackets or extra exemption credits instead of applying single-schedule tax ($2,274.81 on $61,134.44) less $306 of personal and senior credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It applied the joint/surviving-spouse schedule ($1,136) and about $486 of joint plus senior credits. With no dependent child the head files single, which gives $2,274.81 of tax less $306 of credits. It also took the 2% employee-expense floor from CA AGI."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It subtracted only the personal exemption credit and missed the $153 senior exemption credit for age 65+. Its $2,893 also overstates single-schedule tax on $61,134.44, which is $2,274.81."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied qualifying-surviving-spouse rates, but that status requires a dependent child and the head has none. The single schedule applies: $2,274.81 on $61,134.44, less $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It computed tax with the surviving-spouse (joint) schedule, but with no qualifying dependent the head files single. It also subtracted only the personal exemption credit, missing the $153 senior credit."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse schedule ($1,159.12) and a joint $306 personal credit plus $153 senior credit. The no-dependent head files single, which gives $2,274.81 less $153 personal and $153 senior credits. It also took the 2% employee-expense floor from CA AGI."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It subtracted only the personal exemption credit and missed the $153 senior exemption credit that a 77-year-old filer receives. That left its result about $191 above the $1,968.81 obtained after the full $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its $2,095 equals single-schedule tax on about $60,781 less only the $153 personal credit. It missed the $153 senior exemption credit and took the 2% employee-expense floor from CA AGI instead of federal AGI."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse brackets and $474 of credits, but a widow(er) without a dependent child files single. Single filing taxes $61,134.44 at $2,274.81 and allows only $306 of personal plus senior credits."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,missing_output,False,"It declined to compute and defaulted to $0. The facts fully determine CA taxable income of $61,134.44, single-schedule tax of $2,274.81, and $306 of exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It subtracted only the $153 personal exemption credit and missed the $153 senior exemption credit for age 65+. It also took the 2% employee-expense floor from CA AGI, giving $60,781 of taxable income instead of $61,134.44."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got taxable income ($61,134.44) and credits ($306) right but rounded the single-schedule tax to 'about $2,280' instead of computing the exact $2,274.81. That left it $5.19 off."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI ($6,521 instead of $6,167.56), which understated taxable income. Its projected single brackets also understated the tax ($2,173 instead of about $2,247 even on its own $60,781)."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,It said no nonrefundable credits apply and so skipped the $153 personal and $153 senior exemption credits ($306). It also took the 2% employee-expense floor from CA AGI instead of federal AGI.
+us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It misstated CA AGI as $90,623 instead of $91,524.41 and subtracted only a $154 personal credit, missing the $153 senior exemption credit. Together those errors produced $2,024 instead of $2,274.81 − $306."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It left out California's miscellaneous itemized deduction for unreimbursed employee business expenses above 2% of federal AGI ($6,167.56), so it taxed $67,302. It also subtracted only a $144 personal credit and missed the senior exemption credit."
+us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It concluded income falls below the taxable threshold. In fact CA taxable income after excluding Social Security and itemizing $30,390 is $61,134.44, which carries $2,274.81 of tax before $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It wrongly said California taxes federally taxable Social Security and kept CA AGI at $109,149 instead of subtracting $17,624.75. It also left out the $6,167.56 employee-expense deduction and used MFJ brackets for a single filer."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $10,726 standard deduction instead of the $30,390 of itemized deductions, wrongly saying California disallows unreimbursed employee expenses. It also used made-up brackets starting at 4% and $639 of credits instead of the single schedule and $306."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said deductions fully offset CA income. But $91,524 of CA AGI less $30,390 of itemized deductions leaves $61,134.44 taxable, which produces $2,274.81 of tax, and the senior and personal credits offset only $306 of it."
us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly found the tentative tax of about $644, then wiped it out with a $1,000 Saver's Credit and an AOTC. Under Form 8880, the $8,000 403(b) distribution is subtracted from eligible contributions, which brings the Saver's Credit to zero. The AOTC is zero because there are no qualified tuition expenses. The tax is therefore $643.43, not $0."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,other,False,"It computed the $643.50 tentative tax correctly but applied a $1,000 Saver's Credit. Form 8880 subtracts the $8,000 taxable 403(b) distribution in the testing period from the $3,599 of contributions, so eligible contributions and the credit are both zero."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution out of gross income and used the outdated $14,600 standard deduction instead of $16,100, which produced negative taxable income. The correct AGI of $22,534 minus $16,100 leaves $6,434 of taxable income and $643 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It computed the $644 tax correctly but offset it with a $1,500 nonrefundable AOTC. The AOTC is a percentage of qualified tuition and related expenses, and none are listed, so the credit is zero no matter what the enrollment and 1098-T flags say."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It assumed qualified 1098-T expenses and applied the AOTC's nonrefundable portion against the $644 tax. No tuition amount is listed, so qualified expenses are $0 and the AOTC is $0, which leaves $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"After correctly reaching about $644 of tax, it applied up to $1,500 of nonrefundable AOTC. With no qualified education expenses listed, the AOTC is zero and the full $643.43 remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"It explicitly assumed that the 403(b) distributions do not reduce eligible Saver's Credit contributions. That is backwards: Form 8880 requires subtracting testing-period distributions, and the $8,000 distribution reduces the $3,599 of contributions to zero, so the credit is $0 and the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,350 by inflating the pre-OBBBA $15,000 figure, instead of using the enacted $16,100. That raised taxable income to $7,185 and the tax to $719 instead of $643.43. Its AGI and credit treatment were correct."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It claimed that taxable income after the $16,100 deduction is near zero and applied the AOTC to eliminate what remained. In fact, taxable income is $6,434 and the AOTC is zero because no qualified tuition expenses are listed."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"It computed the $644 tentative tax correctly but applied a 50% Saver's Credit on $2,000 of contributions. It missed the Form 8880 rule that subtracts the $8,000 403(b) distribution from contributions, which leaves no eligible contributions."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 of employer-sponsored insurance premiums from wages as a pre-tax exclusion, which cut AGI to $14,446, below the standard deduction. The only pre-tax reduction to wages here is the 401(k) deferral, so AGI is $22,534 and taxable income is $6,434."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It excluded the $8,089 of ESI premiums from wages and also used a pre-TCJA standard deduction of $8,350 instead of the $16,100 OBBBA amount. The two errors partly cancelled and produced $609.60, while the correct figure is 10% of $6,434."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA had expired and applied an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made the higher deduction permanent at $16,100 for 2026, with a personal exemption of zero. Its AGI and zero credits were correct, but taxable income should be $6,434, not $8,935."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It used the 2025 standard deduction of $15,750 instead of $16,100. It then applied a $1,000 Saver's Credit without subtracting the $8,000 403(b) distribution from contributions as Form 8880 requires. That subtraction makes the credit zero and leaves $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"It used an estimated $14,000 for the standard deduction plus a personal exemption instead of the $16,100 standard deduction with no exemption. It then zeroed the tax with a $1,000 Saver's Credit, but the $8,000 403(b) distribution offsets all contributions on Form 8880, so the credit is $0."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted that taxable income is zero after deductions. AGI of $22,534 exceeds the $16,100 standard deduction by $6,434, which produces $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It used the nonrefundable Saver's Credit to offset the remaining tax. Form 8880 reduces contributions by testing-period distributions, and the $8,000 403(b) distribution exceeds the $3,599 of contributions, so the credit is zero."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reached an AGI of $14,446 by excluding the $8,089 of ESI premiums from wages, which is not a reduction to AGI here. It then applied a Saver's Credit that the $8,000 403(b) distribution eliminates on Form 8880. The correct AGI of $22,534 yields $643.43 of tax with no credits."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. Its $0 answer is consistent with applying a nonrefundable credit that does not apply here: the Saver's Credit is zero after the $8,000 distribution offset, and the AOTC is zero without tuition. The $643.43 tax on $6,434 of taxable income therefore stands."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its pre-credit tax of $753.50 implies the stale $15,000 standard deduction instead of $16,100. It then subtracted a $200 Saver's Credit, which the $8,000 403(b) distribution reduces to zero under Form 8880."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It claimed that taxable income is zero after the standard deduction. AGI is $22,534, including the $8,000 taxable 403(b) distribution, and the $16,100 standard deduction leaves $6,434 taxable at 10%."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly zeroed the AOTC and Saver's Credit, but its $754 result implies a $15,000 standard deduction (taxable income of $7,535) instead of the 2026 OBBBA amount of $16,100."
-us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI and credit treatment were correct, including netting the distribution against contributions for the Saver's Credit. However, it estimated the 2026 standard deduction at $15,400 instead of $16,100, which overstated taxable income by $700."
-us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"Its reasoning is self-contradictory. It left the traditional 401(k) deferral in AGI, applied an invented $500 exclusion to the 403(b) distribution, used a $15,000 deduction, concluded $0, and then submitted -$997. Tax before refundable credits cannot be negative, and the correct figure is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It relied on the AOTC to eliminate the remaining tax. The AOTC is computed from qualified tuition expenses, none are listed, and so the credit is zero and $643.43 remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied a partial AOTC offset against the tax on the basis of the enrollment flags alone. With no qualified education expenses, the AOTC is zero, and the tax on $6,434 of taxable income is the full $643.43, not $121."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It offset the tax with the Saver's Credit on the listed contributions. It ignored the Form 8880 requirement to subtract the $8,000 403(b) distribution, which reduces eligible contributions and the credit to zero."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"It used the Saver's Credit to eliminate the tax. The $8,000 taxable 403(b) distribution exceeds total retirement contributions, so eligible Form 8880 contributions are zero and the credit is $0."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $8,089 of employer-plan premiums as pre-tax, which cut AGI to $14,446, below the standard deduction. Wages are reduced only by the 401(k) deferral, so AGI is $22,534 and $6,434 is taxable."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated wages alone as falling under the standard deduction and ignored the $8,000 taxable 403(b) distribution. With that distribution included, AGI is $22,534, which exceeds the $16,100 deduction by $6,434."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a TCJA-sunset deduction of about $8,500 plus a $5,300 personal exemption. OBBBA made the TCJA structure permanent, so the 2026 single standard deduction is $16,100 and the personal exemption is zero. Taxable income is $6,434, not about $8,600."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"It used a standard deduction plus a personal exemption, which put taxable income at about $8,900. It then offset the tax with a $1,000 Saver's Credit even though the $8,000 403(b) distribution nets contributions to zero on Form 8880. The correct result is $16,100 of deduction, no exemption, no credit, and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly zeroed the Saver's Credit and AOTC but assumed post-TCJA law, with an $8,500 standard deduction and a $5,450 personal exemption. Under OBBBA, 2026 uses a $16,100 standard deduction and no exemption."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired, used an $8,500 standard deduction, and applied a 15% bracket. The 2026 law uses a $16,100 deduction and the 10% bracket, which gives 10% of $6,434, or $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 of employer health premiums from wages to reach an AGI of $14,446. Those premiums do not reduce AGI here, so AGI is $22,534 and $6,434 remains taxable after the $16,100 deduction."
-us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score against $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score against $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used a $15,400 standard deduction instead of $16,100 and then applied a $1,500 nonrefundable AOTC. The AOTC is zero because no qualified tuition expenses are listed, so the tax is not eliminated."
-us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"It computed the $644 tentative tax correctly but applied a $1,000 Saver's Credit on the full $2,908 of contributions. Form 8880 subtracts the $8,000 403(b) distribution first, which leaves zero eligible contributions."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It used the $15,750 2025 standard deduction instead of $16,100. It then applied the AOTC's nonrefundable portion, but the AOTC is zero with no qualified tuition expenses, so $643.43 of tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed that income is too low to leave tax after the standard deduction and that the AOTC covers any remainder. In fact, $6,434 of taxable income remains, and the AOTC is zero because there are no qualified education expenses."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly reached about $644 of tax on $6,435 of taxable income, then wiped it out with two credits the household doesn't get. It took a $1,000 Saver's Credit without netting the $8,000 403(b) distribution against contributions under §25B(d)(2), which drops the credit base to $0. It also claimed an AOTC even though qualified tuition is $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,other,False,"The model computed $643.50 of tax correctly and correctly set the AOTC to $0. It then applied a $1,000 Saver's Credit without reducing qualified contributions by the $8,000 403(b) distribution in the testing period, which brings eligible contributions and the credit to $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $8,000 taxable 403(b) distribution out of income entirely and used the outdated $14,600 standard deduction instead of 2026's $16,100. Adding the distribution gives AGI of $22,534 and taxable income of $6,434, or $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model's $0 only works if income is wages alone: $17,443 − $2,778 − $130 = $14,535, which is below the $16,100 standard deduction. It ignored the $8,000 taxable 403(b) distribution, which pushes AGI to $22,534 and taxable income to $6,434, or $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model computed the ~$644 tentative tax correctly, then offset it with a $1,500 nonrefundable AOTC. The AOTC is 100%/25% of qualified tuition and fees, and none are listed, so the credit is $0 and the $643 tax stands."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed the head had qualified 1098-T expenses and applied the nonrefundable AOTC. Enrollment and a 1098-T don't create the credit. With $0 qualified tuition the AOTC is $0, so the $643 tax is not reduced."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied up to $1,500 of nonrefundable AOTC against the ~$644 tax. No qualified tuition or fees are listed, so the AOTC base and credit are $0 and the tax stays at $643."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"The model explicitly assumed the $8,000 403(b) distribution doesn't reduce eligible contributions, which gave a $1,000 Saver's Credit. Under §25B(d)(2), distributions received in the testing period reduce qualified contributions, and $8,000 exceeds the $3,599 contributed, so the credit is $0 and $643 of tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model got AGI and the zero nonrefundable credits right but estimated the 2026 single standard deduction as $15,350 by inflating the 2025 figure. The OBBBA-set 2026 amount is $16,100. That leaves $6,434 of taxable income and $643 of tax, not $719."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misadded gross income and claimed taxable income was near $0 after the standard deduction, when it is $6,434. It then used a nonrefundable AOTC to erase the rest, even though $0 qualified tuition makes the AOTC $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"The model computed ~$644 of tax correctly but applied a 50% Saver's Credit on $2,000 of contributions. It never netted the $8,000 403(b) distribution against contributions as §25B(d)(2) requires, which makes the credit $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer-sponsored insurance premiums from wages as if they were employee pre-tax payroll deductions, cutting AGI to $14,446. Only the 401(k) deferral reduces wages here, so AGI is $22,534. The model also used the stale $15,750 standard deduction instead of $16,100."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $8,089 employer insurance premiums from wages, giving AGI of $14,446 instead of $22,534. It also used an $8,350 pre-TCJA standard deduction instead of the OBBBA 2026 amount of $16,100."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and used an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made TCJA permanent, so 2026 has a $16,100 single standard deduction and no personal exemption, which gives $6,434 of taxable income and $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"The model used the 2025 $15,750 standard deduction instead of $16,100. It then applied a $1,000 Saver's Credit without reducing contributions by the $8,000 403(b) distribution in the testing period, which zeroes out that credit."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"The model used a pre-OBBBA standard deduction plus personal exemption of about $14,000 instead of the $16,100 standard deduction with no exemption. It then claimed a 50% Saver's Credit while ignoring that the $8,000 403(b) distribution wipes out qualified contributions under §25B(d)(2)."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model said taxable income is $0 after deductions. AGI is $22,534, including the $8,000 taxable 403(b) distribution, which exceeds the $16,100 standard deduction by $6,434 and produces $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model used the Saver's Credit to offset all the tax but didn't reduce qualified contributions by the $8,000 403(b) distribution. That distribution exceeds total contributions, so the credit is $0 and $643 of tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer insurance premiums from wages, giving AGI of $14,446 instead of $22,534. It then applied a Saver's Credit that the $8,000 403(b) distribution eliminates under the §25B(d)(2) distribution offset."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model's $0 relies on nonrefundable credits wiping out the tax. With $6,434 of taxable income ($643 of tax), the only candidates are the Saver's Credit, which the $8,000 403(b) distribution reduces to $0, and the AOTC, which is $0 with no qualified tuition. Nothing offsets the $643."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model's $753.50 tentative tax implies the stale $15,000 standard deduction instead of $16,100. It then subtracted a $200 Saver's Credit, which should be $0 because the $8,000 403(b) distribution exceeds total contributions under §25B(d)(2)."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model said taxable income is $0. Wages net of the 401(k) deferral plus the $8,000 taxable 403(b) distribution, less the $130 IRA deduction, give AGI of $22,534, which exceeds the $16,100 standard deduction by $6,434 and produces $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model got AGI and the zero Saver's Credit and AOTC right, but its $754 implies about $7,540 of taxable income, meaning it used the pre-OBBBA $15,000 standard deduction. The 2026 amount is $16,100, which gives $643."
+us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model handled AGI and both credits correctly but estimated the 2026 single standard deduction at $15,400 instead of the OBBBA amount of $16,100. That overstated taxable income by $700 and the tax by $70."
+us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model added back the 401(k) deferral and invented a $500 exclusion on the distribution, then used a $15,000 standard deduction. Its reasoning concluded $0 but it submitted an impossible negative $-997, when federal income tax before refundable credits can't go below zero. The correct path gives AGI of $22,534, taxable income of $6,434, and $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used an AOTC nonrefundable credit to eliminate the remaining tax. The AOTC requires qualified tuition and fees, none are listed, so the credit is $0 and $643 of tax on $6,434 of taxable income remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model applied a partial nonrefundable AOTC based only on enrollment status and appears to have left out the $8,000 403(b) distribution. No qualified tuition is listed, so the AOTC is $0, and the correct $22,534 AGI and $16,100 standard deduction give $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The model used the Saver's Credit to offset the tax without netting the $8,000 403(b) distribution against the listed contributions. Under §25B(d)(2) that distribution reduces qualified contributions to $0, so no credit applies and $643 of tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"The model applied a nonrefundable Saver's Credit that ignores the testing-period distribution reduction. The $8,000 403(b) distribution exceeds the $3,599 of contributions, so the credit is $0 and the $643 tax is unreduced."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer-sponsored insurance premiums as pretax wage reductions, giving AGI of $14,446. Only the $2,778 401(k) deferral reduces wages, so AGI is $22,534 and $6,434 remains taxable after the $16,100 standard deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated income as wages below the standard deduction and ignored the $8,000 taxable 403(b) distribution. Including it gives AGI of $22,534, taxable income of $6,434, and $643 of tax, with no nonrefundable credits available."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed TCJA expired and used an ~$8,500 standard deduction plus a ~$5,300 personal exemption. OBBBA made TCJA permanent, so 2026 has a $16,100 standard deduction and no exemption, giving $643 of tax rather than $860."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"The model assumed a standard deduction plus a personal exemption, though OBBBA made the exemption $0 and set the 2026 standard deduction at $16,100. It then applied a $1,000 Saver's Credit while ignoring that the $8,000 403(b) distribution reduces qualified contributions to $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly zeroed out the Saver's Credit and AOTC but assumed TCJA expired, using a $8,500 standard deduction and a $5,450 personal exemption. Under OBBBA the 2026 single standard deduction is $16,100 with no exemption, which gives $6,434 of taxable income and $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA expired, using an $8,500 standard deduction and pre-TCJA 10%/15% brackets. That gave $14,035 of taxable income and $1,485 of tax, instead of the $16,100 OBBBA standard deduction that gives $6,434 taxed entirely at 10%."
+us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer health premiums from wages, giving AGI of $14,446, which falls below the $16,100 standard deduction. The premiums are not employee pretax deductions here, so AGI is $22,534 and $643 of tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for federal_income_tax_before_refundable_credits. The correct derivation gives $6,434 of taxable income taxed at 10%, or $643, with no nonrefundable credits."
+us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value for federal_income_tax_before_refundable_credits. The correct derivation gives $6,434 of taxable income taxed at 10%, or $643, with no nonrefundable credits."
+us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model used a $15,400 standard deduction instead of $16,100. It then applied a $1,500 nonrefundable AOTC even though no qualified tuition is listed, which makes the AOTC $0 and leaves $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model correctly reached ~$644 of tax and a $0 AOTC but computed the Saver's Credit on $2,908 of contributions without subtracting the $8,000 403(b) distribution received in the testing period. That subtraction drives eligible contributions and the credit to $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model used the stale $15,750 standard deduction and then offset the tax with a nonrefundable AOTC. No qualified tuition or fees are listed, so the AOTC is $0, and the $16,100 standard deduction leaves $643 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model said income is too low to owe tax after the standard deduction and leaned on the AOTC. AGI of $22,534 exceeds the $16,100 standard deduction by $6,434, giving $643 of tax, and the AOTC is $0 with no qualified tuition."
us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly phased the EITC out to $0 and even noted that zero listed tuition gives a $0 AOTC. It then reversed itself by assuming 'typical qualified expenses default' of at least $4,000 and submitted a $1,000 refundable AOTC. With no qualified expenses, the AOTC is $0."
us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the full $2,500 AOTC as refundable. The refundable share is capped at 40% ($1,000). More importantly, the credit base is $0 because no qualified tuition expenses are listed. It also left the $8,000 403(b) distribution out of AGI."
us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model made up a refundable AOTC of about $400 plus a small EITC to reach $522. No qualified education expenses are listed, so the AOTC is $0. Its own AGI of $22,535 is above the roughly $19,540 point where the 2026 childless EITC reaches zero, so the EITC is also $0."
@@ -1673,30 +1736,31 @@ us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical
us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, violating the required submission contract."
us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California CHIP as covering disabled adults up to an extended 266% FPL threshold. It failed to apply the categorical exclusion for a person already Medicaid-eligible through the WORKING_DISABLED_BUY_IN pathway, which makes this head ineligible for CHIP regardless of the CHIP income thresholds."
us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable output for head_chip_eligible. The required result was 0 because Medicaid eligibility under WORKING_DISABLED_BUY_IN categorically excludes the head from CHIP.
-us,scenario_023,head_medicaid_eligible,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Correctly found that MAGI of $22,535 exceeds 138% FPL, then checked only whether the head receives SSI. It never applied the working-disabled buy-in category that covers an employed disabled adult at 1.41x FPL, so it answered not eligible."
-us,scenario_023,head_medicaid_eligible,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Treated the 138% FPL adult-expansion limit as the only Medi-Cal test and ruled the head out because income is about $22,500. It ignored the stated disability and employment, which place the head in the working-disabled buy-in category with a much higher income limit."
-us,scenario_023,head_medicaid_eligible,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Claimed that no other Medi-Cal pathway qualifies once MAGI of $22,535 exceeds 138% FPL. That is wrong: the head is disabled and working, and falls in the working-disabled buy-in category, whose limit easily covers income at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Used only the 138% FPL MAGI expansion test for a single adult and stopped when MAGI of about $22,535 exceeded it. It never considered the working-disabled buy-in category available to this employed disabled head."
-us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Checked the 138% FPL expansion test and the SSI-linked pathway, rejecting both. It skipped the working-disabled buy-in category, which does not require SSI receipt and covers an employed disabled adult at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Applied only the 138% FPL Medicaid expansion threshold to MAGI of $22,535. It never evaluated the disability-based working-disabled buy-in category that makes this working disabled head eligible."
-us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Rejected the 138% FPL expansion group and the SSI non-MAGI pathway, then concluded the head is ineligible. It never tested the working-disabled buy-in category, which covers disabled workers with income well above SSI levels and above 138% FPL."
-us,scenario_023,head_medicaid_eligible,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Computed MAGI of $22,535 correctly and compared it with 138% FPL. It then treated SSI-level countable income as the only disability test. It missed the working-disabled buy-in category, which uses a far higher income ceiling for employed disabled adults, so the head qualifies at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Applied only an adult income limit and concluded the head is ineligible. It never recognized that the head, who is disabled and employed, qualifies under the working-disabled buy-in category, which has a much higher income limit than the adult expansion group."
-us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Noted that the head is disabled but did not treat disability plus employment as grounds for Medicaid, and answered not eligible. The engine places this head in the working-disabled buy-in category, which grants eligibility at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Judged eligibility only against the adult Medi-Cal (138% FPL) income limit using wages plus the 403(b) distribution. It never applied the working-disabled buy-in category, which covers this employed disabled head despite income above the adult limit."
-us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Said the facts establish no separate qualifying category after MAGI exceeded the adult-expansion threshold. The listed disability together with 24 hours a week of paid work establishes exactly that category: the working-disabled buy-in, which makes the head eligible."
-us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Correctly found that MAGI of about $22,535 exceeds the expansion limit, and correctly noted that disability alone does not create SSI-linked eligibility. It then missed the working-disabled buy-in category, which grants Medicaid to disabled adults who are employed without requiring SSI."
-us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Compared the head's MAGI only against California's adult-expansion limit and answered not eligible. It ignored the working-disabled buy-in category that applies to this disabled, employed head at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Stopped after MAGI exceeded the adult-expansion limit, saying no separate coverage pathway is specified. The stated disability and employment qualify the head for the working-disabled buy-in category, which the model never evaluated."
-us,scenario_023,head_medicaid_eligible,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,Relied on the adult MAGI limit and the absence of SSI receipt. It missed that the working-disabled buy-in category needs neither and covers an employed disabled adult with MAGI at 1.41x FPL.
-us,scenario_023,head_medicaid_eligible,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Tested MAGI against 138% FPL and countable income against SSI-related aged/blind/disabled limits, and rejected both. It never applied the working-disabled buy-in category, whose income ceiling sits far above SSI limits and covers this working disabled head."
-us,scenario_023,head_medicaid_eligible,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Limited the analysis to the 138% FPL expansion limit and SSI recipiency. It did not recognize the working-disabled buy-in category, under which the engine finds this employed disabled head eligible."
-us,scenario_023,head_medicaid_eligible,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Rejected the 138% FPL MAGI group and the SSI-level aged/blind/disabled pathway on income. It missed the working-disabled buy-in category, which has a much higher income limit for employed disabled adults and makes the head eligible."
-us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for head_medicaid_eligible, so no answer could be scored. The correct derivation places the working disabled head in the working-disabled buy-in category, which yields eligible (1)."
-us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for head_medicaid_eligible, so no answer could be scored. The correct derivation places the working disabled head in the working-disabled buy-in category, which yields eligible (1)."
-us,scenario_023,head_medicaid_eligible,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Applied only the 138% FPL adult limit and the absence of SSI receipt. It never considered the working-disabled buy-in category that covers this disabled, employed head at 1.41x FPL."
-us,scenario_023,head_medicaid_eligible,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Overstated MAGI at $25,443 by not subtracting the $2,778 traditional 401(k) deferral and the $130 traditional IRA contribution. It then tested only the 138% FPL expansion group and an SSI-level aged/blind/disabled income test. It missed the working-disabled buy-in category, which makes this employed disabled head eligible."
-us,scenario_023,head_medicaid_eligible,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Concluded that household income is above 'the California Medicaid limit', treating the adult-expansion income cap as the only test. It ignored the working-disabled buy-in category, which the head qualifies for as a disabled adult working 24 hours a week."
+us,scenario_023,head_medicaid_eligible,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"The model saw MAGI of about $22,535 above 138% FPL and found no SSI receipt, then said not eligible. It never applied the working-disabled buy-in (California's 250% Working Disabled Program), which covers this employed, disabled head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"The model submitted 0, which contradicts its own explanation saying eligible with value = 1. That explanation also rested on the wrong pathway: it claimed income was well under 138% FPL, but MAGI is 1.41x FPL, so the adult expansion fails. Eligibility comes only from the working-disabled buy-in, which the model never mentioned."
+us,scenario_023,head_medicaid_eligible,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"The model treated the 138% FPL Medi-Cal adult expansion as the only test and ruled the head out because income exceeded it. It ignored that the head is disabled and working, which qualifies them for the working-disabled buy-in at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"The model said no other Medi-Cal pathway applied after MAGI exceeded 138% FPL. That is wrong: the head is disabled and employed, and the working-disabled buy-in (250% FPL limit) covers them at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"The model applied only the 138% FPL single-adult expansion limit. It never considered the working-disabled buy-in category, which covers the disabled, working head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"The model checked only the 138% FPL expansion limit and SSI-linked Medicaid. It missed the working-disabled buy-in, which needs no SSI receipt and covers the disabled, employed head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"The model said not eligible because MAGI exceeded the 138% FPL expansion threshold, and never used the head's disability and employment. Those facts put the head in the working-disabled buy-in category, under whose limits 1.41x FPL qualifies."
+us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"The model looked only at the 138% FPL expansion test and SSI-based non-MAGI pathways. It left out the working-disabled buy-in, which covers this employed, disabled head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"The model correctly found MAGI ($22,535) just above 138% FPL and countable income above the SSI/SSP standard. But it treated SSI-linked coverage as the only disability pathway and never applied California's Working Disabled Program (250% FPL limit, with a $130 asset balance far below any resource limit), under which the head qualifies."
+us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"The model applied only the adult income limit and called the head not eligible. It never evaluated the working-disabled buy-in category, which covers this disabled, working head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"The model brushed aside the stated 'is disabled' fact and gave no income-based reason for ineligibility. A disabled head with wages falls into the working-disabled buy-in category, and 1.41x FPL is within its limits."
+us,scenario_023,head_medicaid_eligible,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"The model compared income only with the adult Medi-Cal (138% FPL) limit. It never applied the working-disabled buy-in, whose higher income ceiling covers the disabled, employed head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no separate qualifying category was established, but the listed facts that the head is disabled and earns wages do establish the working-disabled buy-in category. That category covers the head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"The model concluded that disability without SSI gives no Medicaid pathway. In fact, disability plus employment qualifies the head for the working-disabled buy-in (250% FPL limit), which covers them at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"The model tested only the California adult expansion income limit. It skipped the working-disabled buy-in, which covers the disabled, employed head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no separate pathway was specified, overlooking that the head's disability and wages trigger the working-disabled buy-in. Under that category the head is eligible at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"The model checked only the adult MAGI limit and SSI receipt. The working-disabled buy-in requires neither SSI receipt nor income at or below 138% FPL, and it covers this disabled worker at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"The model rejected both the 138% FPL expansion and the SSI-related aged/blind/disabled pathway on income. It never applied the working-disabled buy-in, whose 250% FPL ceiling covers the employed, disabled head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"The model limited its check to MAGI expansion and SSI receipt. It never considered the working-disabled buy-in, which covers the disabled, working head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"The model ruled out the expansion and the SSI-level aged/blind/disabled pathway on income. It missed the working-disabled buy-in, whose income limit is far above SSI levels and covers the head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,No value or explanation was returned for head_medicaid_eligible. The correct answer is eligible through the working-disabled buy-in category.
+us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,No value or explanation was returned for head_medicaid_eligible. The correct answer is eligible through the working-disabled buy-in category.
+us,scenario_023,head_medicaid_eligible,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"The model applied only the 138% FPL adult limit and the SSI-recipient pathway. It never considered the working-disabled buy-in, which covers the disabled, employed head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"The model overstated MAGI at $25,443 by not subtracting the $2,778 traditional 401(k) deferral. It also rejected the aged/blind/disabled pathway on income without applying the working-disabled buy-in, whose 250% FPL ceiling covers the head at 1.41x FPL."
+us,scenario_023,head_medicaid_eligible,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"The model compared income with the general Medi-Cal adult limit only. It ignored the working-disabled buy-in category, which covers the disabled, working head at 1.41x FPL."
us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the `is_disabled` flag as a stand-in for SSDI-based Medicare eligibility. Medicare for someone under 65 requires 24 months of SSDI entitlement or ESRD, and this head has $0 in Social Security disability benefits and no ESRD. At age 28 with no SSDI history, the head is ineligible, so the correct value is 0."
us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model said disability alone makes someone under 65 eligible for Medicare. It skipped the requirement of 24 months of SSDI entitlement or an ESRD diagnosis, and the head meets neither because they have no SSDI income. The head is 28 and not eligible, so the correct value is 0."
us,scenario_023,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied a disability-based Medicare rule using only the `is_disabled` flag. It ignored that the disability pathway requires 24 months of SSDI entitlement or ESRD, and the head has no Social Security disability income. At age 28 without SSDI or ESRD, the head is ineligible, so the correct value is 0."
@@ -1704,376 +1768,385 @@ us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou
us,scenario_023,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,Correctly included CA SDI on the full wage base but used an invented 1.13% rate ($197.11) instead of the 2026 rate of 1.3% ($226.75). Its submitted 1531.4 also doesn't match its own sum of 1531.50.
-us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Computed only federal FICA ($1,334.39) and left out California's mandatory employee SDI contribution, which is 1.3% of all wages in 2026 ($226.75)."
-us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory employee payroll tax, so it dropped the 1.3% SDI contribution ($226.75). It then inflated its own $1,334.38 FICA total to 1335.85 with no basis."
-us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75), which understates the total by about $17."
-us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"Used the outdated 2023 CA SDI rate of 0.9% ($156.99) instead of the 2026 rate of 1.3% ($226.75). It also submitted 1467.32, which differs from its own computed sum of 1490.38."
-us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"Applied the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75), then rounded its $1,544 total up to 1546."
-us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly kept the full $17,443 wage base but applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75)."
-us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Wrongly assumed California has no mandatory employee payroll tax and left out CA SDI, which is 1.3% of all wages in 2026 ($226.75). It reported FICA alone."
-us,scenario_023,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Explicitly excluded CA SDI, even though the requested payroll_tax output covers mandatory employee state payroll taxes. That dropped the 1.3% SDI contribution ($226.75)."
-us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $8,089 employer-sponsored insurance premiums from the stated gross wages. The FICA base is the full $17,442.65. It also said there is no state payroll tax, which dropped the 1.3% CA SDI."
-us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Shrank the wage base to $9,354 by subtracting the $8,089 employer-sponsored insurance premiums, which do not reduce stated gross wages for FICA or SDI. It also used the 1.2% SDI rate instead of the 2026 rate of 1.3%."
-us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used a combined 8.85% rate, which includes a 1.2% CA SDI rate. The correct combined rate is 8.95%, because the 2026 SDI rate is 1.3%."
-us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,Estimated CA SDI at the 2024 rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Reported only federal FICA and left out CA SDI at 1.3% ($226.75). It also miscalculated Medicare as $252.97 instead of $252.92.
-us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Excluded the $8,089 employer-sponsored insurance premiums from the wage base. Its $818 equals 8.75% (7.65% FICA plus 1.1% SDI) of $9,354. The correct figure applies 7.65% plus the 2026 SDI rate of 1.3% to the full $17,442.65."
-us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Explicitly subtracted $8,089 of employer-sponsored insurance premiums to get a $9,354 base. Stated gross wages are the payroll tax base. It also used the outdated 1.1% SDI rate instead of 1.3%."
-us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Correctly used the full $17,443 wage base but overstated CA SDI. Its $1,595 total implies about $261 of SDI (roughly 1.5%), while the 2026 rate of 1.3% gives $226.75."
-us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 equals 8.65% of $9,354. That means it subtracted the $8,089 employer-sponsored insurance premiums from the wage base and used a 1.0% SDI rate. The correct calculation uses the full $17,442.65 and a 1.3% SDI rate."
-us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,Correctly included CA SDI on full wages but used the 2025 rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory employee payroll tax, which dropped the 1.3% SDI ($226.75). It then changed its own $1,334.39 FICA sum to an unexplained 1,329."
-us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax and so left out CA SDI at 1.3% ($226.75). Its 1375.966 doesn't match its stated 7.65% of $17,443, which is $1,334.39."
-us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out CA SDI at 1.3% ($226.75) and also got the FICA arithmetic wrong. It computed 6.2% of $17,443 as $1,070.10 (should be $1,081.47) and added the parts to 1,208 (should be about $1,323)."
-us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at the 2025 rate of 1.2% instead of the 2026 rate of 1.3%, for a combined 8.85% instead of 8.95% of wages."
-us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Computed only 7.65% FICA and left out California's mandatory employee SDI contribution of 1.3% of wages ($226.75).
-us,scenario_023,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,Estimated CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Used the correct 2026 SDI rate of 1.3% but applied it, along with FICA, to $9,354. That base comes from wrongly subtracting the $8,089 employer-sponsored insurance premiums from the $17,442.65 of gross wages."
-us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Computed only Social Security and Medicare and left out CA SDI at 1.3% of wages ($226.75).
-us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Carried the 2025 CA SDI rate of 1.2% ($209.32) forward to 2026, but the 2026 employee rate is 1.3% ($226.75)."
-us,scenario_023,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,Applied the 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"Explicitly carried forward the 2025 CA SDI rate of 1.2% ($209.32), but the 2026 employee rate is 1.3% ($226.75)."
-us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted the $2,778 traditional 401(k) deferral from the FICA and SDI base. Elective deferrals stay in FICA wages, so the base is the full $17,442.65. It also used the outdated 1.1% SDI rate instead of 1.3%."
-us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Treated the $8,089 employer-sponsored insurance premiums as pre-tax deductions from the $17,443 gross wages, which gave a $9,354 base. It also used 1.2% for SDI instead of the 2026 rate of 1.3%."
-us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no payroll_tax value and no explanation, so there was nothing to score."
-us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"Returned no payroll_tax value and no explanation, so there was nothing to score."
-us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,Summed only employee Social Security and Medicare and left out California's mandatory employee SDI at 1.3% of wages ($226.75).
-us,scenario_023,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,Applied CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75).
-us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Its reasoning worked out a total of $1,526.26 that included CA SDI, though at the outdated 1.1% rate. It then submitted the FICA-only figure of 1334.39, which drops the mandatory SDI contribution entirely."
-us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its $261.65 equals exactly 1.5% of $17,443, so it never applied the 6.2% Social Security and 1.45% Medicare rates that it said it was computing. The correct figure is $1,334.36 of FICA plus $226.75 of CA SDI."
+us,scenario_023,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used 1.13% as the CA SDI rate ($197.11) instead of the 2026 rate of 1.3%, which gives $226.75. That understated the state payroll tax by about $30. The FICA amounts were correct."
+us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($1,334.39) and left out California's mandatory employee SDI. At 1.3% with no cap, SDI adds $226.75 on $17,443 of wages."
+us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and so left out CA SDI. SDI is a mandatory 1.3% withholding on all wages ($226.75). The model also replaced its own $1,334.38 FICA sum with an unexplained $1,335.85."
+us,scenario_023,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate and left out California's mandatory employee SDI. At 1.3% of $17,443, SDI adds $226.75."
+us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). This understated the total by about $17.
+us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Used 0.9% for CA SDI ($156.99), an outdated rate, instead of the 2026 rate of 1.3% ($226.75). It then submitted $1,467.32, which does not match its own $1,490.38 sum."
+us,scenario_023,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). It also rounded its $1,543.71 sum to $1,546 for no stated reason."
+us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Assumed the CA SDI rate was 1.1% for 2026 ($191.87). The 2026 rate is 1.3% ($226.75), so SDI was understated by about $35."
+us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Wrongly treated California as having no mandatory employee state payroll tax and left out CA SDI. SDI is withheld at 1.3% on all wages, which is $226.75 here."
+us,scenario_023,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Explicitly excluded CA SDI, even though the question asks for mandatory employee state payroll taxes. This dropped the $226.75 SDI contribution (1.3% of wages)."
+us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $8,089 employer-sponsored insurance premium from FICA wages, even though the stated $17,443 of gross wages is the payroll-tax base. It also left out CA SDI (1.3%, $226.75) completely."
+us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Reduced the payroll-tax base to $9,354 by subtracting the $8,089 ESI premium, although all taxes apply to the full $17,443 of wages. It also used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%."
+us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Used a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). The FICA amounts were correct.
+us,scenario_023,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied a combined 8.85% rate, which uses the 2025 CA SDI rate of 1.2%. The correct combined rate is 8.95%, with 2026 SDI at 1.3%."
+us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,Estimated CA SDI at 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Counted only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($226.75). It also miscalculated Medicare as $252.97 instead of $252.92.
+us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Excluded the $8,089 ESI premium from the wage base, so taxes were applied to $9,354 instead of $17,443. Its $818 equals 8.75% of $9,354, so it also used a 1.1% SDI rate instead of 1.3%."
+us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Subtracted the $8,089 ESI premium to get a $9,354 base, although all of the payroll taxes apply to the full $17,443 of gross wages. It also used a 1.1% CA SDI rate instead of the 2026 rate of 1.3%."
+us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $1,595 total implies about $261 of CA SDI, roughly a 1.5% rate. That overstates the 2026 SDI rate of 1.3%, which gives $226.75 on top of the $1,334.36 of FICA."
+us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,Applied a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 equals 8.65% of $9,354. That means it subtracted the $8,089 ESI premium from wages and used a 1.0% SDI rate. The correct calculation is 7.65% FICA plus 1.3% SDI on the full $17,443."
+us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,Used 1.1% for CA SDI ($191.87) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,glm-5.2,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and left out the 1.3% SDI ($226.75). It then submitted an unexplained $1,329 instead of its own $1,334.39 FICA total."
+us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax and left out the 1.3% employee SDI ($226.75). Its $1,375.97 is also not equal to 7.65% of $17,443 ($1,334.39), which shows an arithmetic error."
+us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out CA SDI (1.3%, $226.75). It also made arithmetic errors: Social Security on $17,443 is $1,081.47, not $1,070.1, and its own components add to about $1,323, not $1,208."
+us,scenario_023,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Estimated CA SDI at the 2025 rate of 1.2%, for a combined 8.85% rate. With the 2026 SDI rate of 1.3%, the combined rate is 8.95% and SDI is $226.75."
+us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Applied only the 7.65% FICA rate and left out California's mandatory 1.3% employee SDI ($226.75).
+us,scenario_023,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,Estimated CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Used the correct 1.3% SDI rate but subtracted the $8,089 ESI premium from the wage base, applying 8.95% to $9,354. The rate applies to the full $17,443 of gross wages."
+us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Counted only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($226.75).
+us,scenario_023,payroll_tax,grok-4.5,llm_error,state_local_rule,False,Carried the 2025 CA SDI rate of 1.2% ($209.32) into 2026 instead of using 1.3% ($226.75).
+us,scenario_023,payroll_tax,grok-4.6,llm_error,state_local_rule,False,Used 1.2% for CA SDI ($209.32) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,grok-4.7,llm_error,state_local_rule,False,Carried the 1.2% CA SDI rate forward instead of applying the 2026 rate of 1.3%. This understated SDI by about $17.
+us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted the $2,778 traditional 401(k) deferral from FICA wages. Elective deferrals stay subject to Social Security, Medicare and SDI. It also used a 1.1% SDI rate instead of 1.3%."
+us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Applied the taxes to $9,354 after subtracting the $8,089 ESI premium, instead of to the full $17,443 of wages. It also used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%."
+us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not submit a payroll_tax value or any explanation. The correct derivation is FICA of $1,334.36 plus CA SDI of $226.75 at 1.3%, for $1,561.12."
+us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"Did not submit a payroll_tax value or any explanation. The correct derivation is FICA of $1,334.36 plus CA SDI of $226.75 at 1.3%, for $1,561.12."
+us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Counted only federal FICA and left out California's mandatory employee SDI, which is 1.3% of all wages ($226.75)."
+us,scenario_023,payroll_tax,ox-alpha,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75).
+us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Correctly said CA SDI is a mandatory employee payroll tax, but used 1.1% instead of 2026's 1.3%. It then submitted the FICA-only figure of $1,334.39 and dropped SDI completely."
+us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its $261.65 is only about 1.5% of wages, so it effectively counted Medicare alone. It dropped the 6.2% Social Security tax ($1,081.44) and the 1.3% CA SDI ($226.75)."
us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"The model treated the head's disabled flag as making this a SNAP elderly/disabled household, but the head receives no SSI or SSDI, so the model should not have applied a medical deduction or an uncapped shelter deduction. The correct $744 shelter cap leaves net income of about $876/month. The model also submitted $3,936, which does not match its own computed $2,328."
-us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled and used an uncapped shelter deduction (rent plus SUA) plus a medical deduction to bring net income to $0 and award the $298 maximum. The head receives no disability-based benefit, so the shelter deduction is capped at $744, and net income of about $876 gives only $35.20/month."
-us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model applied the federal 130% FPL gross income test and a $2,500 asset limit. It missed that California's broad-based categorical eligibility (TANF non-cash) raises the gross limit to 200% FPL, which $2,120/month passes. It also never applied the $744 capped shelter deduction, which brings net income to about $876, below the $1,304 net limit."
-us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model gave the head elderly/disabled SNAP treatment, deducting medical costs over $35 and taking an uncapped $14,837/year shelter deduction. The head receives no SSI or SSDI, so the $744/month cap applies and no medical deduction is allowed. The model then submitted $3,768, which does not match its own computed $2,529."
-us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled, taking an uncapped $1,167/month shelter deduction and a medical deduction. It also used $121 instead of $290.72 for the 20% earned-income deduction. Without disability-benefit receipt, the shelter deduction is capped at $744, net income is about $876, and the benefit is $35.20/month, not the $2,549/year it submitted."
-us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"The model put California's 200% FPL gross limit at about $2,510/month and concluded that $2,120 fails it. The actual limit for one person is about $2,608, so the household passes. With the $744 shelter cap, net income of about $876 still produces a positive $35.20/month benefit."
-us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model assumed the disabled flag removes the shelter cap and allows a medical deduction, which brought net income to $0 and gave the maximum $298/month. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, which the head does not, so the $744 cap applies and net income is about $876."
-us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model applied the elderly/disabled rules because the head is disabled: it took a $73/month medical deduction and an uncapped $1,224 shelter deduction. The head receives no disability-based benefit, so there is no medical deduction and the shelter deduction is capped at $744. That makes net income about $876 and the benefit $35.20/month, not $194."
-us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model used the 130% FPL gross income test and found the household ineligible. California broad-based categorical eligibility uses a 200% FPL gross screen, which $2,120/month passes, and after the $744 capped shelter deduction net income of about $876 yields a positive benefit."
-us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled, applying a medical deduction and an uncapped shelter deduction to reach about $346 net income. Without receipt of a disability-based benefit, the shelter deduction is capped at $744, net income is about $876, and the monthly benefit is $35.20."
-us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model took an uncapped excess shelter deduction because the head is disabled, which reduced net income to about $446. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies, net income is about $876, and the frozen reference's annual benefit is $461, not $1,995. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled. It took a $747/month medical deduction, which overstates listed medical costs by counting employer premiums, and an uncapped $1,561 shelter deduction to reach $0 net income. The head gets no disability benefit, so no medical deduction is allowed and the shelter deduction is capped at $744, leaving about $876 net income."
-us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied elderly/disabled treatment, with a $98 excess medical deduction and an uncapped $1,239 shelter deduction, to reach $283 net income. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and no medical deduction applies, which gives about $876 net income and a $35.20/month benefit."
-us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model waived the gross test and used the medical and uncapped shelter deductions because the head is disabled, reaching $344 net income. SNAP disabled status requires disability-benefit receipt, so the $744 shelter cap applies, net income is about $876, and the benefit is $35.20/month."
-us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model applied a medical deduction and an uncapped shelter deduction (rent plus SUA) because the head is disabled, which reduced net income to $0 and gave the maximum. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is about $876. The model also used an outdated $291 maximum instead of $298."
-us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"The $3,492 answer equals a $291/month maximum allotment for 12 months, which implies zero net income from an uncapped disabled-household shelter deduction. The head is not SNAP-disabled, since no SSI or SSDI is received, so the $744 shelter cap leaves about $876 net income and a $35.20/month benefit."
-us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model claimed uncapped shelter and medical deductions because of disability and awarded the maximum. SNAP disabled status requires receiving a disability-based benefit, which the head does not, so the shelter deduction is capped at $744, there is no medical deduction, and net income is about $876."
-us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model assumed uncapped shelter and medical deductions for a disabled head, which put net income at about $300/month. The head receives no SSI or SSDI, so the $744 shelter cap applies and net income is about $876. That gives the frozen reference's $35.20/month for January–September, not about $200. The frozen reference's higher October–December amount uses the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model concluded that income exceeds the SNAP threshold. California's 200% FPL broad-based categorical eligibility screen admits $2,120/month gross, and after the earned-income, standard and $744 capped shelter deductions, net income of about $876 passes the net test with a positive benefit."
-us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model applied an uncapped shelter deduction because the head is disabled. The head is not SNAP-disabled without receiving a disability benefit, so the shelter deduction is capped at $744, net income is about $876, and the frozen reference's annual benefit is $461, not $1,980. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated the household as disabled with an uncapped shelter deduction, which reduced net income to $0 and gave the $298 maximum. Without SSI or SSDI receipt, the $744 cap applies and about $876 net income leaves $35.20/month."
-us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model correctly found the student exemption through 24 weekly work hours but wrongly applied uncapped shelter and medical deductions for disability. The head receives no disability-based benefit, so the $744 shelter cap applies and net income is about $876."
-us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"The model exempted the household from the gross test and applied the $35 medical threshold and an uncapped $14,837/year shelter deduction because the head is disabled. SNAP disabled status requires receipt of a disability benefit, so the shelter deduction is capped at $744/month, no medical deduction applies, and net income is about $876/month."
-us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"The model left out the excess shelter deduction entirely, even though rent is $2,000/month, and took 30% of annual net income but compared it against a monthly maximum allotment, which drove its benefit to $0. It then submitted an unsupported $871. The capped $744 shelter deduction brings net income to about $876/month and the frozen reference's benefit to $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model asserted without calculation that income and assets are too high. Assets of $130 are far below any limit, gross income passes California's 200% FPL screen, and the $744 capped shelter deduction reduces net income to about $876, which yields $35.20/month."
-us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model returned $0 on the premise that eligibility was not supported, and never ran the SNAP test. The household is categorically eligible through California's TANF non-cash broad-based categorical eligibility, passes the net income test at about $876 against $1,304, and receives $461 for the year in the frozen reference, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"The model applied medical and uncapped shelter deductions as a disabled household to reach $0 net income and the $298 maximum. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and net income is about $876, giving $35.20/month."
-us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model assumed the disabled head unlocks the medical deduction and an uncapped shelter deduction, producing the maximum allotment. SNAP disabled status requires receiving a disability-based benefit, so the $744 cap applies and the frozen reference's benefit is $35.20/month for January–September 2026, with a higher October–December amount from the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model correctly applied the student work exemption but treated the head as SNAP-disabled, with a medical deduction and an uncapped shelter deduction. Without disability-benefit receipt, the shelter deduction is capped at $744 and net income is about $876, giving the frozen reference's $461/year instead of $2,424. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model concluded that earnings plus the 403(b) distribution leave no benefit, which ignores the excess shelter deduction on $2,000/month rent. Even capped at $744, that deduction brings net income to about $876, and 30% of that ($262.80) is below the $298 maximum."
-us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model applied California's standard medical deduction and an uncapped shelter deduction for disability, reaching $205.80 net income. Both depend on elderly/disabled status, which requires receiving a disability benefit the head does not get, so only the $744 capped shelter deduction applies and net income is about $876."
-us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model treated the disabled head as qualifying for medical and uncapped shelter deductions, which gave the maximum $298/month. The head receives no SSI or SSDI, so the $744 shelter cap applies and net income of about $876 yields $35.20/month."
-us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model used an uncapped shelter deduction because of disability to bring countable income to zero. SNAP disabled status requires disability-benefit receipt, so the shelter deduction is capped at $744, leaving about $876 net income and a $35.20/month benefit."
-us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied an $880 disability medical deduction and an uncapped shelter deduction to reach $3,850/year net income. The head is not SNAP-disabled without SSI or SSDI receipt, so the $744/month cap applies, net income is about $876/month, and the frozen reference's annual benefit is $461, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed income and assets exceed limits after the housing deduction. Assets are only $130, gross income passes California's 200% FPL broad-based categorical eligibility screen, and net income after the $744 capped shelter deduction is about $876, below the $1,304 net limit."
-us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"The model applied the net-income-only test with an excess medical deduction and an uncapped $14,837 shelter deduction because the head is disabled. SNAP disabled status requires receiving a disability benefit, so the shelter deduction is capped at $744/month, no medical deduction applies, and the frozen reference's benefit is $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"The model correctly passed the 200% FPL broad-based categorical eligibility gross test but used excess medical and uncapped shelter deductions for disability to reach $0 net income. The head receives no SSI or SSDI, so the $744 cap leaves about $876 net income. The model also used an outdated $292 maximum instead of $298."
-us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"The model applied an excess medical deduction and an uncapped shelter deduction because the head is disabled, reaching $283 net income. The head is not SNAP-disabled without disability-benefit receipt, so the shelter deduction is capped at $744 and net income is about $876, which gives $35.20/month."
-us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model left the $8,000 403(b) distribution out of income and treated the head as SNAP-disabled, with a medical deduction and an uncapped shelter deduction, to reach $0 net income. Counting the $666.67/month distribution and capping shelter at $744 gives about $876 net income and $35.20/month."
-us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"The model applied a medical deduction and an uncapped shelter deduction for a disabled head to reach about $300 net income. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income is about $876, which gives the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
-us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model exempted the household from the gross test and used medical and uncapped shelter deductions because the head is disabled, which produced $0 net income. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is about $876. The model also used an outdated $292 maximum."
-us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was no answer to score against the $461.34 reference."
-us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"The model used an unexplained $22,535 gross income and applied the federal 130% FPL gross limit. California's broad-based categorical eligibility raises the gross screen to 200% FPL, which the actual $25,443 ($2,120/month) passes, and the net income of about $876 yields a positive benefit."
-us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"The model correctly capped the shelter deduction at $744 and applied the FY2026 $298 maximum to all 12 months, as the release's SNAP convention (c_snap_hold_fy2026) does, but it used the outdated $204 standard deduction instead of $209. The frozen reference's $48.18/month for October–December ($304.68 maximum, $256.50 contribution) uses the engine's projected FY2027 schedule, which is how it reaches $461.34; with the convention's FY2026 schedule all year, the current engine gives $408 ($34 × 12, which also reflects the SNAP rounding fixes) against its $402."
-us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model left out the $8,000 403(b) distribution and the excess shelter deduction, used a $258 maximum allotment, and then awarded the full maximum despite computing $944 net income. The correct figures are $2,120.25 gross, the $744 capped shelter deduction and about $876 net income, which give $35.20/month."
-us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model asserted without calculation that income and resources are too high. Assets of $130 are trivial, gross income passes California's 200% FPL broad-based categorical eligibility screen, and the $744 capped shelter deduction brings net income to about $876, below the $1,304 net limit, which yields the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)."
+us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Treated the 'is disabled' flag as making this a SNAP elderly/disabled household, so it took an excess medical deduction and an uncapped shelter deduction. SNAP disability status requires receiving a disability-based benefit such as SSI or SSDI, which the head does not, so the shelter deduction is capped at $744 and net income is $876.51, not $345. It also submitted $3,936 even though its own calculation reached $2,328."
+us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Applied the medical deduction and the uncapped excess shelter deduction (rent plus SUA) on the theory that the head is disabled, which brought net income to $0 and gave the $298 maximum. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the $744 shelter cap applies, there is no medical deduction, and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Applied the federal 130% FPL gross income test and treated high rent as disqualifying. California's broad-based categorical eligibility (through TANF non-cash assistance) sets the gross limit at 200% FPL, which $2,120/month passes. Rent increases the shelter deduction rather than disqualifying the household, so net income of $876.51 gives $35.20/month, not $0."
+us,scenario_023,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Concluded that income exceeds SNAP limits. In fact the household passes California's 200% FPL broad-based categorical eligibility gross screen, and after the 20% earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction, net income is $876.51. That is below the $1,304.17 net limit and yields $35.20/month."
+us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and applied an uncapped $14,837/year shelter deduction plus a medical deduction, but SNAP disability requires receiving SSI, SSDI or a similar benefit, so the $744/month cap applies and there is no medical deduction. It also submitted $3,768 even though its own calculation reached $2,529."
+us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Applied the medical deduction and an uncapped shelter deduction because the head is 'disabled'. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is $876.51. It also miscomputed the earned-income deduction as $121/month instead of $290.72, then submitted $2,549, which does not match its own $1,788 result."
+us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Misstated California's 200% FPL gross limit as about $2,510 and concluded the household fails, when $2,120/month is under the roughly $2,608 limit. With the $744 shelter cap, net income is $876.51, below the $1,304.17 net limit, so the benefit is $35.20/month rather than zero."
+us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Assumed the disability flag removes the shelter cap and allows a medical deduction, which brought net income to $0. SNAP elderly/disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income of $876.51 yields only $35.20/month."
+us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Took a $73.33 medical deduction and an uncapped $1,223.90 shelter deduction for a 'disabled' head, even though the head receives no disability-based benefit that would make them SNAP-disabled. With the $744 cap and no medical deduction, net income is $876.51. The model also used the outdated $292 maximum allotment and $204 standard deduction instead of $298 and $209."
+us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Applied the federal 130% FPL gross test and ignored California's broad-based categorical eligibility, which sets the gross limit at 200% FPL; the household passes at $2,120/month. Net income after the capped $744 shelter deduction is $876.51, which yields $35.20/month instead of zero."
+us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Waived the shelter cap and added a medical deduction because the head is 'disabled', which gave net income of about $346. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the $744 cap holds, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Used an uncapped excess shelter deduction because the head is disabled, which gave net income of $445.80. SNAP disability status requires receiving a disability-based benefit, so the shelter deduction is capped at $744, giving net income of $876.51 and a benefit of $35.20/month."
+us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Treated the household as SNAP-disabled and used an uncapped $1,560.94 shelter deduction. Its $747.42 medical deduction also counted the $8,089 employer-sponsored premium as an out-of-pocket medical expense. Without SSI or SSDI receipt there is no medical deduction and the shelter deduction is capped at $744, so net income is $876.51."
+us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied a $98.33 excess medical deduction and an uncapped $1,238.90 shelter deduction for a 'disabled' head. SNAP disability requires receiving a disability-based benefit, so the $744 shelter cap applies with no medical deduction, giving net income of $876.51 and $35.20/month."
+us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Waived the gross test and used uncapped shelter and medical deductions because the head is disabled, which gave net income of $344.29. The head is not SNAP-disabled because they receive no SSI or SSDI, so the $744 cap applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Applied a medical deduction and an uncapped shelter deduction (rent plus a $595 SUA) on the theory that the head is disabled, which brought net income to $0, and used a $291 maximum allotment. The household is not SNAP-disabled without SSI or SSDI receipt: the shelter deduction is capped at $744, net income is $876.51, and the benefit is $298 − $262.80 = $35.20/month."
+us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Gave no derivation. Its $3,492 equals 12 × a $291 maximum allotment, which implies zero net income from uncapped shelter deductions. The correct calculation caps the shelter deduction at $744 because the head receives no disability-based benefit, giving net income of $876.51 and $35.20/month."
+us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Claimed that disability makes the shelter deduction uncapped and allows a medical deduction, which brought net income to zero. SNAP's disabled-member definition requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Used uncapped shelter and medical deductions for a 'disabled' head, which gave net income of about $300. Without receipt of a disability-based benefit the household is not SNAP-disabled, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income or resources exceed the limits. The household passes California's 200% FPL gross screen, its $130 in assets is far below any limit, and net income after the $744 capped shelter deduction is $876.51, below the $1,304.17 net limit, which yields $35.20/month."
+us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Applied an uncapped excess shelter deduction because the head is disabled. SNAP's disability definition requires receiving SSI, SSDI or a similar benefit, so the deduction is capped at $744 and net income is $876.51, which yields $35.20/month rather than about $165."
+us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the household as disabled for SNAP and used an uncapped shelter deduction, which brought net income to $0 and gave the $298 maximum. The head receives no disability-based benefit, so the $744 cap applies and net income of $876.51 gives $35.20/month."
+us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Applied uncapped shelter and medical deductions on the basis of the disability flag. SNAP-disabled status requires receiving a disability-based benefit, which the head does not, so the shelter deduction is capped at $744, there is no medical deduction, and net income is $876.51."
+us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Exempted the household from the gross test and took a $1,180 medical deduction and an uncapped $14,836.80 shelter deduction because the head is disabled. Without SSI or SSDI receipt the shelter deduction is capped at $744/month with no medical deduction. It also used the outdated $292 maximum allotment ($3,504/year) instead of $298."
+us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Subtracted a monthly standard deduction from annual income and compared 30% of annual net income to a monthly maximum allotment, which mixes periods and wrongly zeroes the benefit. It also never applied the shelter deduction, then submitted an unsupported $871. The correct monthly calculation gives net income of $876.51 and $35.20/month."
+us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Concluded that income and assets are too high. Gross income of $2,120/month passes California's 200% FPL broad-based categorical eligibility screen, $130 in assets is negligible, and net income after the $744 capped shelter deduction is $876.51, which yields $35.20/month."
+us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Declined to compute SNAP even though the prompt says to assume take-up. The household is eligible through California's 200% FPL broad-based categorical eligibility, and net income of $876.51 after the $744 capped shelter deduction yields $35.20/month."
+us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Used medical and uncapped excess shelter deductions for a 'disabled' household, which brought net income to $0. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the shelter deduction is capped at $744, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied the medical deduction and the uncapped shelter deduction because the head is disabled, which produced the maximum allotment. SNAP disability requires receiving a disability-based benefit, so the $744 shelter cap applies and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Applied an uncapped 'disability shelter' deduction and a medical deduction for the head. Without SSI or SSDI receipt the household is not SNAP elderly/disabled, so the shelter deduction is capped at $744, there is no medical deduction, and net income is $876.51."
+us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Concluded that wages plus the 403(b) distribution leave no benefit. Gross income passes California's 200% FPL screen, and net income after the 20% earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction is $876.51, which leaves a $35.20/month benefit."
+us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Applied California's standard medical deduction and an uncapped shelter deduction because the head is disabled, which gave net income of $205.80. The head receives no disability-based benefit, so neither the medical deduction nor the uncapped shelter deduction is available. The $744 cap gives net income of $876.51 and $35.20/month."
+us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and used rent and medical deductions to reach the maximum $298. SNAP disability requires receiving SSI, SSDI or a similar benefit, so the $744 shelter cap applies, there is no medical deduction, and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Applied an uncapped shelter deduction because of the disability flag, which brought countable income to zero. The head is not SNAP-disabled without receiving a disability-based benefit, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Took an $880 disability-related medical deduction and an uncapped shelter deduction, which gave annual net income of $3,849.60. Because the head receives no SSI or SSDI, neither deduction applies as computed: the shelter deduction is capped at $744/month, net income is $876.51/month, and the benefit is $35.20/month."
+us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed that income and assets exceed SNAP limits. Assets of $130 are trivial, gross income passes California's 200% FPL broad-based categorical eligibility screen, and net income after the $744 capped shelter deduction is $876.51, below the $1,304.17 net limit, which yields $35.20/month."
+us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Applied a $1,180 excess medical deduction and an uncapped $14,837 shelter deduction for a 'disabled' household. SNAP disability requires receiving a disability-based benefit, so the shelter deduction is capped at $744/month with no medical deduction. It also used the outdated $292 maximum allotment ($3,504/year)."
+us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Correctly passed the 200% FPL broad-based categorical eligibility test but used medical and uncapped shelter deductions, which brought net income to $0, and an outdated $292 maximum allotment. The head is not SNAP-disabled without SSI or SSDI receipt, so the $744 cap applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Applied an excess medical deduction on $1,600 of costs and an uncapped shelter deduction, which gave net income of about $283. Without receipt of a disability-based benefit the household is not SNAP-disabled: the $744 shelter cap applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Left the $8,000 taxable 403(b) distribution out of countable unearned income and applied uncapped shelter and medical deductions for a 'disabled' head. The correct gross income is $2,120.25/month, and with the $744 shelter cap and no medical deduction, net income is $876.51."
+us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"Used the medical deduction and an uncapped shelter deduction for the disabled head, which gave net income of about $300. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and net income is $876.51, which yields $35.20/month."
+us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"Exempted the household from the gross test and applied uncapped shelter and medical deductions because of the disability flag, which brought net income to $0, and used a $292 maximum. SNAP disability requires receiving a disability-based benefit, so the $744 cap applies and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"Returned no answer or explanation for snap, so there was no value to score against the $461.34 reference."
+us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted the traditional 401(k) and IRA contributions to get $22,535 of gross income, then applied the federal 130% FPL gross test. California's broad-based categorical eligibility uses a 200% FPL gross limit, which the $25,443 gross income passes, and net income of $876.51 yields $35.20/month."
+us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Correctly applied the $744 shelter cap but used the FY2025 $204 standard deduction instead of $209. It also applied FY2026 parameters to all 12 months, missing the October–December 2026 increase to a $304.68 maximum allotment with net income of $855, which raises those three months to $48.18 each."
+us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Left the $8,000 403(b) distribution out of countable income, applied no shelter deduction on $2,000/month rent, and then set the benefit equal to the maximum allotment despite positive net income. The correct net income is $876.51, which gives $298 − $262.80 = $35.20/month."
+us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that income and resources are too high. Gross income passes California's 200% FPL broad-based categorical eligibility screen, assets are only $130, and net income after the $744 capped shelter deduction is $876.51, below the net limit, which yields $35.20/month."
us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable."
us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found AGI of $22,535, a standard deduction of about $5,700, tax of about $200, and both the ~$154 exemption credit and the $60 renter's credit. It then called the tax 'effectively 0' instead of subtracting: $225.78 − $153 − $60 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction of about $5,880 instead of the $5,706 published for 2025. That understated taxable income and tax (about $222 instead of $225.78) and produced $9 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ignored the $8,000 taxable 403(b) distribution and the 401(k)/IRA adjustments, and used a $5,202 standard deduction. It also subtracted CalEITC as if it were nonrefundable, but CalEITC is refundable and does not reduce this pre-refundable-credit amount."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model understated the tax at about $170 (the correct figure is $225.78, using $5,706 and a 1% bracket ending at $11,079) and never applied the $60 renter's credit. It then reported $0 even though its own $170 − $149 left a positive balance."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed about $233 in gross tax and a ~$149 exemption credit, and admitted the tax was 'not fully eliminated'. It then asserted $0 anyway, left out the $60 renter's credit, and never did the subtraction ($225.78 − $153 − $60 = $12.78)."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model claimed about $200 of tax was 'fully offset' by a ~$150 exemption credit, which is an arithmetic contradiction. It also left out the renter's credit. The correct result is $225.78 − $213 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model applied inflated 2026 projections: a ~$5,880 standard deduction and a ~$158 exemption credit. The published $5,706 and $153 apply. That left about $1 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model added a second ~$144 exemption credit for disability. California's extra exemption credit is for blindness, not disability, so the head gets only the single $153 credit. The doubled credit wiped out the tax, and the model also used outdated parameters ($5,540 deduction, $10,756 bracket)."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model estimated $250–$300 of tax, then claimed a ~$149 exemption credit exceeded it and cited nonexistent 'disability-related' credits. Only the $153 exemption credit and the $60 renter's credit apply, and together they leave $12.78 of the $225.78 tax."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model correctly reached $226 of tax and the $153 exemption credit but left out the $60 nonrefundable renter's credit. The household pays $24,000 in rent and is under the income limit, so the credit applies. The model got $73 and rounded to $75 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution. That cut taxable income to $8,906 and the tax below the exemption credit. The correct AGI is $22,534, which gives $225.78 of tax and leaves $12.78 after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution, so its $90.83 tax was wiped out by the exemption credit. Including the distribution gives $16,828 taxable and $225.78 of tax, and the $213 in credits leave $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used inflated 2026 projections: a $5,876 standard deduction, a 1% bracket ending at $11,044, and a $158 exemption credit. The published $5,706, $11,079, and $153 apply. That understated the residual as $4.74 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's AGI of $22,665 skipped the $130 traditional IRA deduction. It also used an outdated $5,540 standard deduction and a 1% bracket ending at $10,756. That overstated tax at $234.94 and produced $20.94 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a rounded ~$5,800 deduction, ~$230 of tax, and a ~$155 exemption credit instead of the exact $5,706, $225.78, and $153. That gave $15 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model claimed the single standard deduction exceeds AGI, but the $5,706 deduction is far below the $22,534 CA AGI. That leaves $16,828 of taxable income and $225.78 of tax before credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model asserted that the personal exemption credit alone fully offsets the tax. The $153 credit covers only part of the $225.78 tax, and even with the $60 renter's credit, $12.78 remains."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $8,846 shows it left the $8,000 taxable 403(b) distribution out of CA AGI. The correct taxable income is $16,828, which gives $225.78 of tax and $12.78 after the $213 in credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $0 matches leaving out the $8,000 403(b) distribution or assuming the credits exceed the tax. The correct figures are $225.78 of tax on $16,828 taxable, less $213 in credits, for $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model's $218 is roughly the tax before credits ($225.78). It never subtracted the $153 personal exemption credit or the $60 nonrefundable renter's credit, which bring the tax down to $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model claimed the standard deduction and exemption credits eliminate the tax. The $5,706 deduction leaves $16,828 taxable and $225.78 of tax, which is more than the $153 exemption and $60 renter's credits combined, so $12.78 remains."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the right structure (1%/2% brackets, exemption credit, $60 renter's credit) with approximate parameters. It got $17 instead of the $12.78 that the published $5,706 deduction, $11,079 bracket, and $153 credit produce."
-us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated California's personal exemption as a $138 deduction from income, but it is a $153 credit against tax. It also said no credits apply, omitting both that credit and the $60 renter's credit, and used outdated parameters ($5,363, $10,234). The result was $238.34."
-us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $2,778 traditional 401(k) deferral and the $130 IRA deduction, putting AGI at $25,443 instead of $22,534. It also never applied the $153 personal exemption credit, subtracting only the $60 renter's credit."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model asserted the deduction and credits wipe out the tax. The $225.78 tax on $16,828 of taxable income is more than the $153 exemption and $60 renter's credits combined, leaving $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model's $72 matches $225.78 minus only the $153 exemption credit, so it left out the $60 nonrefundable renter's credit. It also cited an American Opportunity credit, which California does not have."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model added a 2.5% California early-distribution additional tax on the $8,000 403(b) distribution. That penalty is not part of this output, and nothing in the facts makes the distribution early. The model also wrongly claimed the regular tax was eliminated, when $225.78 − $153 − $60 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model claimed the exemption and renter's credits fully offset the tax. Those credits total $213, which is less than the $225.78 tax, leaving $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a 'projected' 2026 standard deduction that was inflated above the published $5,706. That understated tax and gave $4 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model claimed the exemption and renter's credits fully offset the tax. The $153 + $60 = $213 in credits is less than the $225.78 tax, leaving $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model applied an 'indexed' 2026 standard deduction above the published $5,706. That shrank taxable income and tax, giving $4 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model used the right structure but estimated parameters and rounded to a whole dollar, giving $13. The exact calculation with $5,706, $11,079, and $153 is $225.78 − $213 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,The model used estimated 2026 parameters above the published 2025 values: a higher standard deduction and bracket threshold. That understated tax and left $3 instead of $12.78.
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution, so the exemption credit appeared to wipe out the tax. The correct CA AGI is $22,534, which gives $225.78 of tax and $12.78 after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed zero tax even before credits. The $5,706 standard deduction leaves $16,828 of taxable income and $225.78 of tax, and the $213 in credits reduce that only to $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used inflated 2026 projections: a $5,690 standard deduction, inflated brackets, and a $158 exemption credit. The published $5,706, $11,079, and $153 apply. It got $9 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model itemized a $7,499 medical deduction by counting employer-sponsored premiums (paid pre-tax) and OTC expenses. The deductible medical costs ($800) fall below the 7.5%-of-AGI floor (about $1,690), so the $5,706 standard deduction applies and the tax is $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used inflated 2026 estimates: a $5,843 deduction, a 1% bracket ending at $11,345, and a $157 exemption credit. The published $5,706, $11,079, and $153 apply. It got $3 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model applied only the $60 renter's credit and left out the $153 personal exemption credit that every single filer receives. That left $169 instead of $12.78.
-us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an AGI of about $14,446, leaving out the $8,000 taxable 403(b) distribution, so the exemption credit appeared to wipe out an $87 tax. Including it gives $225.78 of tax and $12.78 after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score against $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score against $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added back the traditional 401(k) and IRA amounts, but California conforms to the 401(k) exclusion and the IRA deduction, so CA AGI is $22,534. It also said no credits apply, omitting the $153 exemption and $60 renter's credits."
-us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used an outdated $5,540 standard deduction and a $149 exemption credit instead of the published $5,706 and $153. That overstated taxable income and understated the credit, giving $20 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model added a second $144 exemption credit for disability, but California's extra exemption credit applies only to blindness. It also skipped the 401(k)/IRA adjustments and wrongly treated the renter's credit as refundable. Together these zeroed out a tax that is really $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted the standard deduction and nonrefundable credits eliminate the tax. The $225.78 tax on $16,828 of taxable income is more than the $213 in exemption and renter's credits, leaving $12.78."
-us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It cut earned income to $14,665 by subtracting the 401(k) deferral, even though CalEITC earned income is the full $17,443 of wages. It also never ran the phase-out on the $22,534 AGI. The childless schedule at that AGI gives $95.15, not $402."
-us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It phased the credit out using only earned income of $17,443. It ignored that CalEITC's phase-out uses the greater of earned income and AGI, and AGI here is $22,534 because of the $8,000 in 403(b) distributions. Applying the schedule at $22,534 lowers the credit from about $180 to $95.15."
-us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It said a filer with no qualifying dependents cannot get CalEITC. In fact, CalEITC covers filers without children who are 18 or older and have earned income under roughly $32,000, and this 28-year-old with $17,443 in wages qualifies for $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Its own reasoning estimated the childless CalEITC at $0–$50. It then replaced that with an unsupported $358 that 'reflects interactions', even though no other California refundable credit applies. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It reported $1,208, several times the childless CalEITC maximum of about $300, even while calling the credit 'small'. It never computed the phase-out at AGI $22,534, which gives $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It claimed the childless CalEITC is fully phased out below this income. The childless income limit is roughly $32,000, so at AGI $22,534 the filer is still in the phase-out range and gets $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"It correctly phased the credit out using AGI of about $22.5k. But it misapplied the second-segment phase-out rate, landing at $115 instead of $95.15."
-us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It treated the $8,000 in taxable 403(b) distributions as disqualifying investment income. That test covers only interest, dividends, rents, royalties, capital gains, and passive income, and pension or retirement distributions are not included. The filer stays eligible and gets $95.15."
-us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It gave roughly the maximum childless CalEITC ($285) and applied no phase-out. At the $22,534 AGI that governs the phase-out, the credit drops to $95.15."
-us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It placed the filer on the phase-out slope using only earned income of $17,443. It ignored the rule that the greater of AGI ($22,534) and earned income sets the phase-out, which cuts the credit to $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It used made-up figures of $6,576 earned income and $14,446 AGI, when the real figures are $17,443 and $22,534. That put the filer near the plateau with about $300 of credit, instead of deep in the phase-out where the credit is $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It invented a rule that CalEITC equals 85% of the federal EITC. CalEITC has its own schedule, and the federal childless EITC is fully phased out at AGI $22,534 anyway. The correct CalEITC is $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It concluded that no CalEITC applies. A childless filer aged 28 with $17,443 in wages and $22,534 AGI is under the roughly $32,000 childless limit and gets $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It used one invented straight-line phase-out from $15,000 to $33,000, when CalEITC actually phases out in two segments, and it left the $130 IRA deduction out of AGI. That produced $172.25 instead of $95.15 at AGI $22,534."
-us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It correctly phased the credit out using the higher AGI of $22,535. But it rounded to $100 instead of computing the exact value from the schedule, which is $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It said income rules out any California refundable credit. The childless CalEITC runs to roughly $32,000, so this filer still gets $95.15 at AGI $22,534."
-us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It phased the credit out only partially, getting $148. Running the full two-segment schedule at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Its $180 matches a phase-out computed from earned income of $17,443 alone. It skipped the greater-of-AGI-or-earned-income rule, and with AGI at $22,534 the credit falls to $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 and never recognized that a childless filer aged 28 with wages qualifies for CalEITC. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It said the filer's income exceeds the CalEITC limits. The childless limit is roughly $32,000, well above both the $17,443 in earnings and the $22,534 AGI, so the credit is $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $195 matches a phase-out based on wages alone. Applying the phase-out at the higher $22,534 AGI cuts the credit to $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It correctly phased the credit out but approximated the schedule, getting $92. Evaluating the exact childless CalEITC schedule at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It claimed the childless CalEITC disappears before AGI $22,535. The childless income limit is roughly $32,000, so the credit is $95.15 at this AGI."
-us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It said the filer fails the childless age test. For CalEITC, the only age requirement for a childless filer is being at least 18, which this 28-year-old meets, so the credit is $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no California refundable credit, missing that the filer's $17,443 in wages makes them eligible for the childless CalEITC. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It found no California refundable credit, missing that the filer's $17,443 in wages makes them eligible for the childless CalEITC. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It correctly placed the filer in the childless phase-out range but misestimated the second-segment slope. It got $114 instead of the $95.15 the schedule gives at AGI $22,534."
-us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It based the credit explicitly on $17,443 of earned income. It skipped the rule that the greater of AGI ($22,534) and earned income drives the phase-out, which lowers the credit to $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It computed the credit 'at $17,443 of earned income'. It ignored that the higher AGI of $22,534, raised by the 403(b) distributions, sets the phase-out, and that gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It said the listed facts generate no California refundable credit, missing that the childless CalEITC covers this 28-year-old wage earner. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It correctly found a phased-out childless CalEITC but pushed the phase-out too far, getting $25. The schedule at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly phased the credit out using the higher AGI but misestimated the phase-out slope, getting $106. The exact value at AGI $22,534 is $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It said a childless adult at this income gets no CalEITC. The childless limit is roughly $32,000, so the filer is still in the phase-out range at AGI $22,534 and gets $95.15."
-us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It applied a phase-out that was too steep, getting $28. The childless CalEITC schedule at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It found no qualifying refundable credits, missing that the childless CalEITC applies to this 28-year-old wage earner. The phase-out at AGI $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"It made CalEITC depend on federal EITC eligibility. CalEITC is separate from the federal credit and extends to roughly $32,000 for childless filers, so it pays $95.15 even though the federal childless EITC is zero at this AGI."
-us,scenario_023,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It reasoned that a $0 federal EITC means a $0 CalEITC. CalEITC has its own schedule with a childless limit of roughly $32,000, which gives $95.15 at AGI $22,534."
-us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly phased the credit out at AGI $22,535 but used the wrong phase-out rate, getting $120 instead of the $95.15 the schedule gives."
-us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It gave roughly the maximum childless CalEITC ($300) and applied no phase-out. At the governing AGI of $22,534, the credit is $95.15."
-us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no parseable value for state_refundable_credits, so the $95.15 CalEITC was never reported."
-us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no parseable value for state_refundable_credits, so the $95.15 CalEITC was never reported."
-us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed CalEITC is nonrefundable for filers without children. CalEITC is fully refundable whether or not the filer has children, so this filer's $95.15 counts as a state refundable credit."
-us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It said $14,665 of earned income is above the childless phase-out ceiling. The ceiling is roughly $32,000, and earned income is actually $17,443, so the filer gets $95.15 at AGI $22,534."
-us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It counted California's $60 renter's credit as refundable, but that credit is nonrefundable. It also inflated CalEITC to $251, contradicting its own $200 figure, without phasing it out at AGI $22,534, where the only refundable credit is $95.15."
-us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It said no California refundable credit applies, missing the childless CalEITC this 28-year-old wage earner qualifies for. The phase-out at AGI $22,534 gives $95.15."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model built AGI (~$22,535) and the standard deduction (~$5,700) correctly but put the tax at about $200 instead of $225.78 (1% of $11,079 plus 2% of the remaining $5,749). It then declared the $213 of credits a full offset, even though $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction of ~$5,880 instead of the 2025 amount of $5,706. That cut taxable income to ~$16,655 and the tax to ~$222, so it got $9 instead of $225.78 − $213 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used wages alone, so it left out the $8,000 taxable 403(b) distribution, skipped the 401(k) and IRA adjustments, and used an outdated $5,202 standard deduction. It then subtracted CalEITC as if it were nonrefundable; CalEITC is a refundable credit and does not belong in this output."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed California taxable income is zero, but CA AGI of $22,534 (including the $8,000 403(b) distribution) far exceeds the $5,706 standard deduction. That leaves $16,828 taxable and $225.78 of tax before the $213 of credits."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the older $5,540 standard deduction, put the tax at ~$170 (the 1%/2% schedule on its own ~$17,000 gives ~$240), and subtracted only a ~$149 exemption credit plus an undefined disability adjustment. It concluded $0 instead of $225.78 − $153 − $60 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own figures (~$233 tax minus a ~$149 exemption credit) leave a positive balance, yet it called the remaining tax effectively $0. It also never applied the $60 renter's credit. The correct net is $225.78 − $213 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model understated the tax as ≈$200 (it is $225.78) and asserted that a ~$150 exemption credit fully offsets it. The $153 exemption credit plus the $60 renter's credit total $213, which leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model projected 2026 parameters: a ~$5,880 standard deduction and a ~$158 exemption credit. The applicable values are the 2025 ones, $5,706 and $153, which give $225.78 of tax and $12.78 after credits, not $1."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model invented a second $144 exemption credit for disability. California's personal exemption credits are for the taxpayer, blind status, and dependents, with no disability credit. It also used the 2024 standard deduction and brackets ($5,540 and $10,756) and omitted the renter's credit. With the correct $153 + $60 credits, $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model guessed $250–$300 of tax, then claimed the ~$149 exemption credit exceeds it, which contradicts its own numbers. The correct $225.78 tax minus the $153 exemption credit and the $60 renter's credit leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"The model computed the tax (~$226) and the $153 exemption credit correctly but left out the $60 nonrefundable renter's credit. This filer qualifies for it: they pay $24,000 in rent and AGI is under the single-filer limit. It also rounded $73 up to $75."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model's CA AGI of $14,446 leaves out the $8,000 taxable 403(b) distribution, which is included in California AGI. Including it gives $22,534 AGI, $16,828 taxable income, and $225.78 of tax, which the credits do not fully offset."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a CA AGI of $14,446, which omits the $8,000 taxable 403(b) distribution, together with an outdated $5,363 standard deduction. It therefore taxed only ~$9,083 at 1%, when the correct taxable income is $16,828.34 with $225.78 of tax."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model projected 2026 indexing: a $5,876 standard deduction, a $11,044 bracket threshold, and a $158 exemption credit. The applicable values are the 2025 ones ($5,706, $11,079, and $153), which give $12.78 rather than $4.74."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model left out the $130 traditional IRA deduction (AGI $22,665 instead of $22,534) and used the 2024 standard deduction ($5,540) and 1% bracket top ($10,756) instead of 2025's $5,706 and $11,079. That inflated the tax to $234.94 and the result to $20.94."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model approximated the standard deduction (~$5,800 vs $5,706), the tax (~$230 vs $225.78), and the exemption credit (~$155 vs $153), then rounded. The exact 2025-parameter calculation gives $225.78 − $213 = $12.78, not $15."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model claimed the single standard deduction exceeds AGI, but the $5,706 deduction is far below the $22,534 CA AGI. That AGI includes wages net of the 401(k) deferral plus the $8,000 403(b) distribution."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model asserted that the personal exemption credit fully offsets the tax. The $153 credit covers only part of the $225.78 tax, and even with the $60 renter's credit $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $8,846 leaves out the $8,000 taxable 403(b) distribution. With it included, taxable income is $16,828.34 and the tax of $225.78 exceeds the $213 of nonrefundable credits."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"With no derivation, the model's $0 implies it treated the tax as fully offset. The correct calculation is $22,534 AGI minus the $5,706 standard deduction, giving $225.78 of tax, less the $153 exemption and $60 renter's credits, for $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,The $218 answer is roughly the tax before credits: the model never subtracted the $153 personal exemption credit or the $60 renter's credit. Both are nonrefundable and reduce the $225.78 tax to $12.78.
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model claimed the standard deduction and exemption credit eliminate the tax. Taxable income is $16,828.34 with $225.78 of tax, and the $153 exemption plus $60 renter's credit leave $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied the right structure (1%/2% brackets, exemption credit, $60 renter's credit) but used approximate parameters instead of the 2025 values. The exact values ($5,706 deduction, $11,079 bracket, $153 credit) give $12.78, not $17."
+us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated California's personal exemption as a $138 deduction from income instead of a nonrefundable credit (2025: $153). It said no nonrefundable credits apply, which skipped both that credit and the $60 renter's credit. It also used the outdated 2023 $5,363 deduction and $10,234 bracket."
+us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used $25,443 as CA AGI, failing to exclude the $2,778 traditional 401(k) deferral and deduct the $130 IRA contribution, both of which California conforms to. It also left out the $153 nonrefundable personal exemption credit and subtracted only the $60 renter's credit."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model asserted the deduction and credits wipe out the tax without computing it. The $225.78 tax on $16,828 of taxable income exceeds the $213 of exemption and renter's credits by $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The $72 answer equals the $225.78 tax minus only the $153 exemption credit, so the $60 nonrefundable renter's credit was never applied. The American Opportunity Credit the model cites is a federal credit and does not reduce California tax."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model added a 2.5% California early-distribution additional tax ($200) that is not part of this output. It also claimed the regular tax is fully eliminated, when $225.78 − $213 leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model claimed the exemption and renter's credits fully offset the tax. Together they are $213, against $225.78 of tax on $16,828.34 of taxable income, which leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction and indexed parameters instead of the 2025 values ($5,706 deduction, $11,079 bracket, $153 exemption credit). This understated the tax relative to the credits and gave $4 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model claimed the exemption and renter's credits fully offset the tax. The $153 + $60 credits total $213, below the $225.78 tax, so $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model applied an indexed 2026 standard deduction and exemption credit instead of the 2025 amounts ($5,706 and $153). That gave $4 rather than $225.78 − $213 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The model rounded its estimate to a whole $13. The exact calculation, $225.78 of tax less $153 and $60 of nonrefundable credits, is $12.78, with no rounding to whole dollars."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model used projected 2026 indexed values for the standard deduction, brackets, and exemption credit instead of the 2025 amounts ($5,706, $11,079, and $153). That pushed the result to $3 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model's AGI of $14,446 leaves out the $8,000 taxable 403(b) distribution. California AGI is $22,534, which produces $225.78 of tax that the exemption credit does not eliminate."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed zero tax before credits, but the $5,706 standard deduction leaves $16,828.34 taxable out of $22,534 of CA AGI. That produces $225.78 of tax, and $12.78 remains after the $213 of credits."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used an inflated 2026 exemption credit (~$158 instead of $153) and projected bracket thresholds instead of the 2025 values. That gave $9 instead of $12.78.
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model counted the $8,089 of employer-sponsored insurance premiums as an itemized medical deduction. The filer's deductible medical costs ($300 of premiums plus $500 of other medical expenses) fall below the 7.5%-of-AGI floor (~$1,690), so the $5,706 standard deduction applies and $12.78 of tax remains."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model projected 2026 parameters ($5,843 standard deduction, $11,345 bracket, $157 exemption credit) instead of the 2025 amounts ($5,706, $11,079, and $153). That reduced the tax to ~$220 and the result to $3 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,The model subtracted only the $60 renter's credit and never applied the $153 nonrefundable personal exemption credit that every single filer receives. That left $169 instead of $225.78 − $213 = $12.78.
+us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's AGI of ~$14,446 leaves out the $8,000 taxable 403(b) distribution. That understated taxable income at ~$8,720 instead of $16,828.34, so the tax looked smaller than the exemption credit."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $12.78 reference."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $12.78 reference."
+us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added back the traditional 401(k) deferral and IRA deduction, but California conforms to both, so CA AGI is $22,534, not $25,443. It also said no nonrefundable credits apply, which skipped the $153 personal exemption credit and the $60 renter's credit."
+us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model paired the 2024 standard deduction ($5,540) and exemption credit (~$149) with a 2025-era bracket threshold. The 2025 values are a $5,706 deduction and a $153 credit, which give $225.78 − $213 = $12.78 rather than $20."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model invented a $144 'disabled' exemption credit that California does not provide, and it used $25,443 of AGI without the 401(k) and IRA adjustments. It then labeled the renter's credit refundable, when California's renter's credit is nonrefundable and is part of the $213 that leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model asserted zero tax without computing it. $16,828.34 of taxable income produces $225.78 of tax, and the $153 exemption plus $60 renter's credits leave $12.78."
+us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It cut CalEITC earned income to ~$14,665 by subtracting the 401(k) deferral, but CalEITC earned income here is the full $17,443 of wages. It also left out that the phase-out runs on the higher AGI of $22,534. Its $402 is above the childless maximum of about $300, while the AGI-based phase-out gives $95.15."
+us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It ran the childless phase-out on earned income of $17,443 and got ~$180. It missed that when AGI ($22,534, including the $8,000 403(b) distribution) is higher than earnings, the CalEITC phase-out uses AGI, which lowers the credit to $95.15."
+us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It ruled the filer out of CalEITC for having no dependents. CalEITC is open to childless filers aged 18+ with earnings under ~$32k, so this 28-year-old with $17,443 of wages qualifies for $95.15."
+us,scenario_023,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It said CalEITC does not apply without qualifying children. California pays a refundable CalEITC to childless workers aged 18+, which for this filer is $95.15 after the AGI-based phase-out."
+us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It first put the childless CalEITC at ~$0-$50, then raised it to $358 by citing unnamed 'interactions' with no rule behind them. Applying the CalEITC phase-out to the higher AGI of $22,534 gives a single credit of $95.15, and no other California refundable credit applies."
+us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its $1,208 is about four times the childless CalEITC maximum (~$300) and matches a with-children schedule. The childless schedule, phased out on the higher AGI of $22,534, gives $95.15."
+us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated the childless CalEITC as fully phased out below $17,443, as if the old low-income cap still applied. Since the extension, the CalEITC phases out gradually to about $32k, so at AGI $22,534 a $95.15 credit is left."
+us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"It correctly used AGI of about $22.5k as the phase-out measure, but its estimated CalEITC schedule was off and gave $115 instead of $95.15."
+us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It counted the $8,000 taxable 403(b) distribution as investment income and said it broke the CalEITC investment-income limit. Retirement-plan distributions are not investment income for the EITC or CalEITC; they only raise AGI, which lowers the credit to $95.15."
+us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It gave a near-maximum childless CalEITC of $285 and applied no phase-out. At AGI of $22,534, well into the phase-out range, the credit is $95.15."
+us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It estimated the credit on the phase-out slope using earned income of $17,443 alone. The phase-out uses the higher AGI of $22,534, which brings the CalEITC down to $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It used made-up figures of $6,576 earned income and $14,446 AGI, then applied a near-maximum $300 credit. Earned income is $17,443 and AGI is $22,534, and the phase-out on AGI leaves $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the old rule that CalEITC equals 85% of the federal EITC. The current CalEITC has its own schedule and does not depend on the federal credit. The federal childless EITC is also zero at this AGI, so its $503.06 base was wrong as well; the CalEITC schedule phased out on AGI gives $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It said no CalEITC applies. A childless 28-year-old with $17,443 of wages and AGI of $22,534 is within the CalEITC's ~$32k limit and gets $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It used an invented single linear phase-out from $15,000 to $33,000 off a $300 maximum, and a miscalculated AGI of $22,665. The actual CalEITC schedule, applied to AGI of $22,534, gives $95.15, not $172.25."
+us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It correctly ran the CalEITC phase-out on the higher AGI of $22,535 but rounded to $100 instead of computing the exact 2026 schedule, which gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It said the filer's income was too high for any California refundable credit. The childless CalEITC runs to about $32k, so at AGI of $22,534 the filer still gets $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Its $148 matches a phase-out run on W-2 earnings of $17,443. CalEITC uses the greater of earnings or AGI, and the higher AGI of $22,534 gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Its $180 matches running the childless CalEITC phase-out on earned income of $17,443. The phase-out uses the higher AGI of $22,534, which gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,It returned $0 with no reason given and missed that this childless filer aged 18+ with earnings under ~$32k qualifies for a refundable CalEITC of $95.15.
+us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It said the filer's income exceeded the CalEITC limit. The 2026 childless CalEITC runs to roughly $32k, so AGI of $22,534 still yields $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $195 matches a phase-out on wages of $17,443, or no phase-out at all. Running the phase-out on the higher AGI of $22,534 gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It correctly found a phased-out childless CalEITC but estimated the 2026 schedule instead of computing it. At AGI of $22,534 the credit is $95.15, not $92."
+us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It thought the childless CalEITC was fully gone before AGI of $22,535. The CalEITC's final phase-out runs to about $32k, so a $95.15 credit remains at this AGI."
+us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It said the filer failed a childless age test. The CalEITC minimum age is 18 with no upper limit, so this 28-year-old qualifies and gets $95.15 after the AGI-based phase-out."
+us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no California refundable credit and missed the childless CalEITC, which this filer qualifies for with wages of $17,443 and AGI under ~$32k. The credit is $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It found no California refundable credit and missed that the CalEITC covers childless workers aged 18+, which gives this filer $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It placed the filer in the 2026 CalEITC phase-out range but estimated the schedule loosely. Running the phase-out on AGI of $22,534 gives $95.15, not $114."
+us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It computed the childless CalEITC on earned income of $17,443 alone. Because AGI ($22,534) is higher, the phase-out uses AGI, which lowers the credit to $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It figured the CalEITC at $17,443 of earned income. The phase-out uses the higher AGI of $22,534, which gives $95.15 instead of $160."
+us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It produced no California refundable credit and missed that the childless CalEITC covers wages of $17,443 with AGI under ~$32k, giving $95.15."
+us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It recognized a phased-out childless CalEITC but phased it out too far. At AGI of $22,534 the credit is $95.15, not $25."
+us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly ran the phase-out on the higher AGI but estimated the 2026 schedule, overshooting to $106. The exact CalEITC at AGI of $22,534 is $95.15."
+us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It said a single childless adult at this income gets no CalEITC. The childless CalEITC phases out gradually to about $32k, so at AGI of $22,534 the filer gets $95.15."
+us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It applied a childless phase-out that was too steep and got $28. The 2026 CalEITC schedule at AGI of $22,534 leaves $95.15."
+us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It found no qualifying California refundable credit and missed the childless CalEITC, which this filer earns at $95.15 with AGI of $22,534."
+us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It made CalEITC depend on federal EITC eligibility. CalEITC has its own schedule running to about $32k, so it pays $95.15 here even though the federal childless EITC is zero."
+us,scenario_023,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"It reasoned that a zero federal EITC means a zero CalEITC. The CalEITC is computed separately from the federal credit, with its own ~$32k limit, and gives $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly ran the phase-out on AGI of $22,535 but estimated the 2026 CalEITC schedule and got $120. The exact credit is $95.15."
+us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It gave about the childless CalEITC maximum ($300) and applied no phase-out. With AGI of $22,534 well into the phase-out, the credit drops to $95.15."
+us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC."
+us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC."
+us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It said the CalEITC is nonrefundable for childless filers. The CalEITC is fully refundable for all eligible filers, including childless ones aged 18+, and is $95.15 here."
+us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It said earnings of $14,665 were far above the childless phase-out ceiling. The 2026 CalEITC ceiling is about $32k, and on the correct measures ($17,443 earnings, $22,534 AGI) the credit is $95.15."
+us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It added the $60 renter's credit, which is nonrefundable and does not count. It also overstated the CalEITC at $251, which even contradicts its own ~$200 estimate. The only refundable credit is the CalEITC, phased out on AGI of $22,534 to $95.15."
+us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It said no California refundable credit applies and missed the childless CalEITC, which this 28-year-old earner qualifies for at $95.15."
us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached about $7,019, using a $32,300 standard deduction that is $100 too high (the 2026 MFJ figure is $32,200). It then submitted $3,325, which matches none of its computation. The submitted value contradicts the bracket math it had just done."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $29,900 MFJ standard deduction instead of the 2026 OBBBA amount of $32,200, which overstated taxable income at $65,025. Its $8,640 is not even consistent with that taxable income under 10%/12% brackets (about $7,370). That means it also misapplied the rate schedule."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $32,300 standard deduction instead of $32,200. It also ended the 10% bracket at $24,000 instead of the 2026 MFJ threshold of $24,800. The two parameter errors mostly cancel and give $7,035 rather than $2,480 + 12% × $37,925.30 = $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It applied the correct $32,200 standard deduction and the $24,800 10% bracket and reached about $7,031. It then submitted $6,934 through an unexplained 'rounded with bracket estimates' adjustment that has no basis in the tax computation."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deliberately fell back to the pre-OBBBA 2025 parameters: a $30,000 MFJ standard deduction and a 10% bracket ending at $23,850. The 2026 law sets $32,200 and $24,800. Those figures inflated taxable income to $64,925 and the tax to $7,314."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed the correct figure: $62,725 of taxable income after the $32,200 deduction, and tax of $2,480 + $4,551 = $7,031. It then submitted $4,700 through an unexplained 'adjusting for standard deduction estimate' step that discards its own correct result."
-us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 MFJ standard deduction and ended the 10% bracket at $24,150. The 2026 values are $32,200 and $24,800. This overstated taxable income at $64,325 and gave $7,236."
-us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 MFJ standard deduction of $30,000 and the 2025 10% bracket top of $23,850. The 2026 figures are $32,200 and $24,800. This overstated taxable income by $2,200 and gave $7,314."
-us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA expired after 2025 and applied a pre-TCJA $16,700 standard deduction, personal exemptions and a 15% second bracket. The OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no personal exemptions and a 12% bracket. That produced $8,906.25."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from AGI even though the head has no wages to exclude them from. It imputed $14,640 of mortgage interest from the loan balance, although unlisted amounts are zero. It also itemized SALT and applied expired personal exemptions and a 15% bracket. The correct approach is the $32,200 standard deduction against $94,925.30 of AGI under 10%/12% brackets."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $22,400 standard deduction instead of the 2026 MFJ amount of $32,200, which overstated taxable income at $72,525. Its $5,236 is also inconsistent with that taxable income under the brackets (about $8,200), so it misapplied the rate schedule too."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly applied a TCJA-expiration baseline, with the pre-TCJA standard deduction, personal exemptions and the 15% bracket. The OBBBA made the $32,200 standard deduction and 12% bracket permanent for 2026. That inflated the tax to $9,112."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: a $16,100 standard deduction, $10,100 of personal exemptions and a 15% bracket above $23,200. The 2026 OBBBA regime has a $32,200 deduction, no exemptions and a 12% bracket above $24,800. That produced $9,148.75 instead of $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted that the standard deduction and 'applicable credits' reduce tax to $0. The $32,200 standard deduction leaves $62,725.30 of taxable income, and this childless couple under 65 has no nonrefundable credits to offset the resulting $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no parameters, but $8,993 is about $1,960 above the $7,031.04 that the $32,200 deduction and 10%/12% brackets produce. That matches the TCJA-sunset computations other models made, with a pre-TCJA deduction, personal exemptions and a 15% bracket, rather than 2026 OBBBA law."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It explicitly computed the tax 'under sunset rules' with personal exemptions. It did not use the OBBBA-permanent 2026 structure: a $32,200 MFJ standard deduction, no exemptions and a 12% bracket above $24,800. The result was $7,306.75."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 MFJ standard deduction of $30,000 and the 2025 10% bracket top of $23,850 instead of the 2026 figures of $32,200 and $24,800. That overstated taxable income at $64,925 and gave $7,314."
-us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,900 and the 10% bracket top at $24,600. The actual figures are $32,200 and $24,800, so taxable income came out at $64,025 and the tax at $7,191."
-us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It used a $30,600 standard deduction instead of $32,200, which gave $64,325 of taxable income and $7,223 of tax, and it correctly concluded that no nonrefundable credits apply. It then submitted $6,005.50, which contradicts its own final figure of $7,223."
-us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It claimed the standard deduction and unspecified credits eliminate all tax. The $32,200 MFJ standard deduction leaves $62,725.30 taxable, and no nonrefundable credit applies to this childless couple under 65, so the tax is $7,031.04, not $0."
-us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $2,460 equals 10% of about $24,600, which is only the 10%-bracket tax. It left out the 12% tax on the $37,925.30 of taxable income above the $24,800 bracket threshold, a further $4,551.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that no taxable income remains after deductions. The $32,200 MFJ standard deduction against $94,925.30 of AGI leaves $62,725.30 taxable, which produces $7,031.04 of tax."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-expiration law: a $16,688 standard deduction, $10,644 of personal exemptions and a 15% second bracket. The OBBBA made the $32,200 standard deduction and 12% bracket permanent for 2026, so it overstated taxable income at $67,593 and the tax at $8,914."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the TCJA-sunset regime: a $16,600 standard deduction, two $5,300 exemptions and a 15% bracket above $24,800. The OBBBA-permanent 2026 regime has a $32,200 deduction, no exemptions and a 12% bracket, so it produced $8,919."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,700 and the 10% bracket top at $24,450. The actual figures are $32,200 and $24,800, which gave $64,225 of taxable income and $7,218 of tax."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted' pre-TCJA law: a $16,800 standard deduction, $10,800 of personal exemptions and a 15% bracket above $23,837. The OBBBA-permanent 2026 schedule has a $32,200 deduction, no exemptions and 12% above $24,800, so it produced $8,907."
-us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It estimated $5,500–$6,500 of tax on $63,425 of taxable income, using a $31,500 deduction instead of $32,200, and acknowledged that no nonrefundable credits apply. It then submitted $0 anyway. That discards a positive liability that no credit offsets."
-us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $31,450 MFJ standard deduction and a $24,350 10% bracket top instead of the 2026 values of $32,200 and $24,800. That overstated taxable income at $63,475 and gave $7,130."
-us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"It subtracted a $3,346.75 'senior bonus' from the tax for the 61-year-old head. The OBBBA senior deduction requires age 65, and it is a deduction, not a tax credit. It also used a $9,300 standard deduction instead of $32,200, a 10% bracket top of $29,550 instead of $24,800, and an itemized medical deduction the standard deduction would displace."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning used a $32,300 standard deduction ($100 too high) and reached about $7,019, close to the correct $7,031. It then submitted $3,325, a number that none of its own steps support. The final value does not match its own bracket calculation."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $29,900 MFJ standard deduction instead of the 2026 amount of $32,200. It also mis-computed the tax: $8,640 on $65,025 implies far more than the 10%/12% rates produce. The correct result is $62,725 taxable, giving $2,480 + 12% × $37,925 = $7,031."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It put taxable income at about $84,000, so it never properly subtracted the $32,200 MFJ standard deduction from $94,925 of AGI. Its final $5,600 figure does not follow from either taxable-income amount through the 10%/12% brackets."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $32,300 standard deduction instead of $32,200 and ended the 10% bracket at $24,000 instead of the 2026 MFJ threshold of $24,800. The two parameter errors nearly cancelled, giving $7,035 rather than $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It applied the correct $32,200 standard deduction and $24,800 10% bracket and computed $7,031. It then arbitrarily changed the answer to $6,934, calling it rounding with bracket estimates, and submitted that unsupported figure."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 parameters for a 2026 return: a $30,000 standard deduction and a 10% bracket ending at $23,850. The 2026 OBBBA-indexed values are $32,200 and $24,800. Its taxable income was therefore $64,925 instead of $62,725."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its reasoning correctly computed $62,725 of taxable income and $2,480 + $4,551 = $7,031 of tax. It then submitted $4,700 after an unexplained ""adjusting"" step that no rule supports."
+us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,600 instead of $32,200 and ended the 10% bracket at $24,150 instead of $24,800. These errors overstated taxable income by $1,600 and produced $7,236."
+us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the 2025 MFJ standard deduction ($30,000) and 10% bracket end ($23,850) to tax year 2026. The 2026 standard deduction of $32,200 leaves $62,725 taxable, and the 10% bracket runs to $24,800."
+us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired after 2025, so it brought back personal exemptions, a $16,700 standard deduction and a 15% second bracket. The OBBBA made the TCJA structure permanent: a $32,200 standard deduction, no exemptions and a 12% second bracket."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It made three errors. It subtracted the head's $21,208 of ESI premiums from AGI, even though the head has no wages and the spouse's wages are given as gross. It made up $14,640 of mortgage interest from the loan balance, although unlisted amounts are treated as 0. It also applied pre-TCJA personal exemptions and a 15% bracket instead of the permanent $32,200 standard deduction and 12% rate."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $22,400 standard deduction instead of the 2026 MFJ amount of $32,200, giving $72,525 of taxable income. Its $5,236 tax does not even match 10%/12% rates applied to that taxable income."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly used a post-TCJA-expiration 2026 baseline with personal exemptions and the old standard deduction. The OBBBA made the TCJA permanent, so the correct calculation uses the $32,200 MFJ standard deduction and 10%/12% brackets, giving $7,031."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied sunset-law parameters: a $16,100 standard deduction, $10,100 of personal exemptions and a 15% bracket above $23,200. Under the OBBBA, 2026 uses a $32,200 standard deduction, no exemptions, and 12% above $24,800."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It reported $0 even though $94,925 of AGI minus the $32,200 standard deduction leaves $62,725 taxable. No nonrefundable credit applies to this childless couple under 65, so the tax is $7,031."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $8,993 is consistent with the pre-TCJA schedule: a smaller standard deduction plus exemptions, and 15% above the 10% bracket. The OBBBA-permanent 2026 rules use a $32,200 deduction and 12% up to $100,800, giving $7,031."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied TCJA sunset rules with personal exemptions for 2026. The OBBBA made the TCJA standard deduction and bracket structure permanent, so 2026 uses a $32,200 MFJ deduction and 10%/12% rates."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 MFJ standard deduction of $30,000 and the 2025 10% bracket end of $23,850. It should have used the 2026 values of $32,200 and $24,800, which overstated taxable income by $2,200."
+us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,900 instead of $32,200 and ended the 10% bracket at $24,600 instead of $24,800. Its taxable income was $64,025 instead of $62,725."
+us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction instead of $32,200 and reached $7,223. It then submitted $6,005.50, which matches neither its own computation nor any credit it found, since it ruled out the elderly credit for a couple under 65."
+us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and credits eliminate all tax. In fact $94,925 of AGI minus the $32,200 standard deduction leaves $62,725 taxable, and no nonrefundable credit applies to this childless couple under 65."
+us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $2,460 is consistent with taxing only the 10% bracket (about $24,600 × 10%). It omits the 12% tax on the remaining $37,925 of taxable income, which is $4,551."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted there was no taxable income after deductions. The $32,200 MFJ standard deduction leaves $62,725 of taxable income, which produces $7,031 of tax with no offsetting nonrefundable credits."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA expiration in 2026, using a $16,688 standard deduction, $10,644 of personal exemptions and a 15% second bracket. The OBBBA made the $32,200 standard deduction and 12% bracket permanent."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA sunset parameters: a $16,600 standard deduction, two $5,300 exemptions, and 15% above $24,800. Under the OBBBA, 2026 uses a $32,200 standard deduction, no exemptions, and a 12% second bracket."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,700 instead of $32,200 and ended the 10% bracket at $24,450 instead of $24,800. Its taxable income was $64,225 instead of $62,725."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used reverted pre-TCJA rules: a $16,800 standard deduction, $10,800 of personal exemptions and 15% above $23,837. The OBBBA-permanent 2026 rules use a $32,200 standard deduction, no exemptions, and 12% above $24,800."
+us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there is nothing to score. The correct derivation is $94,925 AGI minus the $32,200 standard deduction = $62,725 taxable, giving $7,031.04 of tax."
+us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It estimated $5,500 to $6,500 of tax on $63,425 of taxable income, using a $31,500 standard deduction instead of $32,200. It then reported $0 on the reasoning that having no credits makes the tax zero, which reverses the logic: no nonrefundable credits means the full $7,031 remains."
+us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $31,450 MFJ standard deduction instead of $32,200 and a 10% bracket ending at $24,350 instead of $24,800. Its taxable income was $63,475, giving $7,130 instead of $7,031."
+us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"It used a $9,300 standard deduction instead of $32,200 and a wrong $29,550 10% bracket. It then subtracted a ""senior bonus deduction"" as a tax credit for the 61-year-old head, but the OBBBA senior deduction requires age 65 or older and reduces taxable income rather than tax."
us,scenario_025,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_025,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model correctly stated that the 61-year-old head does not meet Medicare eligibility, but then submitted value = 1, which denotes Yes. It inverted its own eligibility conclusion when encoding the numeric output."
-us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the correct FICA components ($3,888.95 + $909.51) and then added a fabricated ""Ohio state payroll tax (employee share)"" of ~$621 by applying a 0.99% Ohio income-tax rate to wages — Ohio has no mandatory employee-side payroll tax, and state income tax belongs to state_income_tax_before_refundable_credits, not payroll_tax. It then compounded that with an invented ""$13.22 OASDI contribution cap adjustment"" and a final unexplained $4,000 jump from its own $5,432.76 subtotal to the submitted $9,432.76."
-us,scenario_025,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its rule application was exactly right — 6.2% and 1.45% on the spouse's $62,725 only, no Additional Medicare Tax, no Ohio employee payroll tax — and it stated the correct total of $4,798.46 in its explanation. It then discarded that figure for an invented ""$4,800.14 based on precise calculation,"" a $1.68 fabrication that corresponds to no rate, base, or wage-cap computation and contradicts the explanation it submitted alongside it."
-us,scenario_025,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the head's $21,208 employer-sponsored insurance premium from the spouse's $62,725 wages, treating ESI as a section 125 pre-tax exclusion from the FICA wage base and yielding a $41,517 base. PolicyEngine's employee_social_security_tax and employee_medicare_tax apply to gross wages with no ESI offset, and the $21,208 premium belongs to the head, who has zero wages, so it cannot reduce the spouse's base in any event; 7.65% of the correct $62,725 is $4,798.48."
-us,scenario_025,payroll_tax,glm-5.3,llm_error,other,False,"It derived the answer correctly — $62,725 under the 2026 wage base, 6.2% = $3,888.95 plus 1.45% = $909.51, no Additional Medicare Tax, head with no wages — and wrote $4,798.46 in its explanation, but submitted 4795.60 in the outputs object. The $2.86 gap matches nothing in its derivation; it is a transcription error between the computed figure and the value field."
-us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its explanation names the right base and rates (6.2% plus 1.45% on the spouse's $62,725, no Additional Medicare Tax), which yields $4,798.48, but the submitted $9,571.50 is roughly double that — consistent with applying the combined employer-plus-employee 15.3% FICA rate instead of the 7.65% employee share. The prompt explicitly excludes employer payroll taxes, so only the employee 7.65% belongs in this output."
-us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It stated the tax was ""computed on annual wages for both spouses,"" but the head has no gross wages listed and unlisted numeric inputs are zero, so the base is the spouse's $62,725 alone. Its $9,106 corresponds to 7.65% of roughly $119,000 — nearly twice the actual base — from imputing wages to the wageless head and/or sweeping in the spouse's $32,200 private pension, which is not FICA-taxable; the correct product is $62,725 × 7.65% = $4,798.48."
-us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the submission carries no substantive computation to evaluate. The required derivation is 6.2% plus 1.45% on the spouse's $62,725 of gross wages — the only FICA wages in the household — for $4,798.48."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used $2,150 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It applied 2.75% above $26,050 with no $332 base amount. It then made an unexplained cut from about $1,576 down to $875, when the correct figure is $2,121.57 minus the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The exemptions ($3,800), taxable income ($91,125) and $200 retirement credit were right. The model left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess. Pre-credit tax was therefore $1,789.56 instead of $2,121.57."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted a federal standard deduction, but Ohio starts from federal AGI and has no standard deduction. It also used an outdated 0.501%–5.75% bracket schedule instead of the $332 + 2.75% bracket. It skipped the $1,900 exemptions and the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used $1,850 exemptions instead of $1,900. It added an arbitrary amount of about $144 instead of the $332 base. It also said no nonrefundable credits applied, which misses the $200 retirement income credit on the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model got $91,125 of taxable income but computed only 2.75% × $65,075 = $1,790, leaving out the $332 base amount. It then gave $2,426, which is higher than its own pre-credit tax even though it said credits were subtracted. It also mentioned a joint filing credit that does not apply because the head has no qualifying income."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used $2,500 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It used a $360 base amount instead of $332. The credits it subtracted do not match the single $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model assumed the 2026 schedule has no base amount. It left out the $332 fixed tax at the start of the 2.75% bracket, so pre-credit tax was $1,789.56 instead of $2,121.57, and the answer after the $200 credit was $1,589.56 instead of $1,921.57."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It also left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess before subtracting the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used $2,350 exemptions and left out the $332 base amount. It subtracted about $500 in credits, including a senior credit that requires age 65 (the head is 61), instead of the single $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Taxable income ($91,125) and the $200 credit were right. The model applied only 2.75% above $26,050 and left out the $332 base amount, which understates the answer by $332."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used $1,000 exemptions instead of $1,900. It invented a 2.5%/3.5% schedule with no $26,050 zero bracket, when the correct bracket is $332 + 2.75% over $26,050. It also skipped the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model took only one $2,400 exemption instead of two $1,900 exemptions. It used an outdated multi-bracket schedule (3.226%/3.688%) instead of $332 + 2.75% over $26,050. It also said no credits applied, missing the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used $2,800 exemptions instead of $1,900. It left out the $332 base amount and skipped the $200 retirement income credit for the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used $2,400 exemptions and applied 3.5% above $46,100 instead of $332 + 2.75% on everything up to $100,000. It took a 5% joint filing credit, which requires both spouses to have qualifying income, and missed the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the head's $21,208 employer insurance premiums from wages as if pre-tax, but the head has no wages and the stated wages are gross. It took an Ohio medical deduction for health premiums, which is not allowed when employer-subsidized coverage is available, and the other medical costs are below 7.5% of AGI. It also used $2,400 exemptions and left out the $332 base amount."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model showed no work. $1,695 matches 2.75% × ($94,925 − $26,050) − $200, which skips the $3,800 exemptions and the $332 base amount. The correct result is $332 + 2.75% × $65,075 − $200 = $1,921.57."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied only 2.75% above $26,050 with no $332 base amount, and its $1,762 implies $4,800 of exemptions instead of $3,800. It did not subtract the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $2,700 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It left out the $332 base amount and skipped the $200 retirement income credit for the pension."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model showed no derivation. $1,280.50 is below even the 2.75%-only amount on $91,125 of taxable income, so the answer includes neither the $332 base amount nor the correct taxable income. The correct result is $2,121.57 minus the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The exemptions ($3,800) and $200 retirement credit were right. The model applied only 2.75% above $26,050 and left out the $332 base amount, which understates the tax by $332."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied 2.75% to the full $94,925 above $26,050 without subtracting the two $1,900 exemptions. It also left out the $332 base amount, while correctly subtracting the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model claimed personal exemptions are $0 above $80,000 of Ohio MAGI, when they are $1,900 each at this income. It left out the $332 base amount and skipped the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted a $30,000 federal standard deduction, which Ohio does not allow, and used $2,400 exemptions. It applied 2.75% to all of taxable income, ignoring both the $26,050 zero bracket and the $332 base amount, and skipped the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used an outdated pre-reform schedule (2.85%–4.396% brackets about $5,600 wide) and a joint filing credit that is unavailable because the head has no qualifying income. Its submitted $1,260.80 does not match its own computed $3,245, and neither matches $332 + 2.75% × $65,075 − $200."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated the household as low-income with no liability. Ohio taxable income of $91,125 is far above the $26,050 zero bracket and produces $2,121.57 of tax before the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction, but Ohio has none. It should have subtracted only the two $1,900 exemptions and applied $332 + 2.75% over $26,050 before the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $360.69 base amount (an earlier year's schedule) instead of $332. It also took a $230.63 medical deduction that counted $6,500 of health premiums, which are not deductible because both spouses have employer-subsidized coverage."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of $1,900 and left out the $332 base amount. It said no nonrefundable credits applied, missing the $200 retirement income credit on the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Taxable income ($91,125) and the $200 credit were right. The model used the older $360.69 base amount instead of $332, which overstates the answer by $28.69."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model excluded the $32,200 private pension from Ohio income. Ohio taxes private pensions and offers only the $200 retirement income credit. The model also used $2,150 exemptions and applied a joint filing credit, which the household does not qualify for because the head has no qualifying income."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model deducted $21,388.63 of medical costs, counting the head's $21,208 employer insurance premiums and the $6,500 of other premiums. Ohio allows no premium deduction when employer-subsidized coverage is available, and the $850 of other expenses is below 7.5% of AGI. It also used $2,150 exemptions instead of $1,900."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied only 2.75% above the zero bracket and left out the $332 base amount. Its answer is the 2.75%-only figure less the $200 credit, instead of $2,121.57 − $200."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The exemptions and retirement credit were right. The model computed only 2.75% of the excess over $26,050 and left out the $332 base amount in the bracket."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The $332 base amount, $3,800 exemptions and $200 credit were right. The model wrongly subtracted $6,500 of health premiums, which Ohio does not allow as a deduction when the taxpayer can get employer-subsidized coverage (both spouses have employer insurance)."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model claimed Ohio deductions and credits reduce tax to zero. With $91,125 of taxable income, well above the $26,050 zero bracket, tax is $2,121.57 before the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of $1,900 and left out the $332 base amount. It also claimed zero nonrefundable credits, missing the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model left out the $332 base amount. It also took a 5% joint filing credit, which requires each spouse to have qualifying income, while the head has none."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Exemptions, the $200 credit, and denial of the joint filing credit were right. The model left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used $2,650 exemptions and applied 3.5% above $46,100, when the correct bracket is $332 + 2.75% over $26,050 up to $100,000. It also skipped the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used $4,800 of exemptions instead of $3,800 and left out the $332 base amount. It said no nonrefundable credits applied, missing the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model used $2,500 exemptions instead of $1,900 and left out the $332 base amount. It also skipped the $200 retirement income credit for the pension."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The $1,900 exemptions and $200 credit were right. The model computed only 2.75% of the excess over $26,050 and left out the $332 base amount, so pre-credit tax was $1,789.56 instead of $2,121.57."
-us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model invented senior or age-based adjustments that zero out the tax. Neither spouse is 65, and Ohio has no age exclusion that removes $91,125 of taxable income from the $332 + 2.75% bracket."
-us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied no personal exemptions, when two $1,900 exemptions apply, and left out the $332 base amount. It missed the $200 retirement income credit on the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model gave no derivation. $2,651 is higher than even the correct $2,121.57 pre-credit tax, which fits an outdated multi-bracket schedule rather than $332 + 2.75% over $26,050, and it did not subtract the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $9,300 Ohio standard deduction and taxed the first $26,050 at 2.75%, though that bracket is 0%. It applied 3.5% above that instead of $332 + 2.75%, and its bracket arithmetic is internally wrong. It claimed a senior credit that requires age 65 and missed the $200 retirement credit."
+us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed $3,888.95 in Social Security tax and $909.51 in Medicare tax on the spouse's wages. It then added Ohio income tax on wages (~$621) as if it were a 'state payroll tax', though Ohio has no mandatory employee payroll tax. It also added made-up 'OASDI cap adjustments', and the $9,432.76 it submitted does not match even its own $5,432.76 sum."
+us,scenario_025,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model's reasoning gets the right answer: the spouse's $62,725 in wages at 7.65% is about $4,799, and Ohio has no state payroll tax. But it submitted $6,000, which contradicts its own computation."
+us,scenario_025,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly summed $3,888.95 + $909.51 = $4,798.46 on the spouse's wages, then switched to $4,800.14 with an invented 'rounding based on precise calculation' step. No payroll tax rule produces that extra $1.68."
+us,scenario_025,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the head's $21,208 employer-sponsored insurance premium from the spouse's $62,725 wages as a pre-tax deduction and applied 7.65% to $41,517. That premium belongs to the head, who has no wages, and nothing in the facts reduces the spouse's FICA wages. The full $62,725 is the FICA base."
+us,scenario_025,payroll_tax,glm-5.3,llm_error,other,False,"The model's reasoning correctly arrives at $4,798.46: 6.2% plus 1.45% on the spouse's $62,725, with no Additional Medicare Tax and no head wages. But it submitted $4,795.60, which conflicts with its own stated result."
+us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model said only the spouse's $62,725 of wages is taxed at 6.2% + 1.45%, which yields $4,798.46. Its $9,571.50 is about double that. This fits applying the combined 15.3% employer-plus-employee FICA rate instead of the 7.65% employee share."
+us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed payroll tax on 'annual wages for both spouses' and reported $9,106. The head has no listed wages, and the spouse's pension is not FICA wages. The correct base is only the spouse's $62,725, giving $4,798.48 at 7.65%."
+us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no parseable payroll_tax value and no reasoning. The correct derivation is 7.65% of the spouse's $62,725 wages, or $4,798.48."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Used $2,150 exemptions instead of the $1,900 that applies above $80,000 MAGI. Computed tax as 2.75% of the excess over $26,050 without the 2026 schedule's roughly $332 base amount, and skipped the $180.60 Ohio unreimbursed medical deduction. It then cut its own ~$1,576 figure to $875 with an unexplained 'graduated structure' adjustment."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the $1,900 exemptions and the $200 retirement income credit right. However, it taxed only 2.75% of income over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction ($7,300 less 7.5% of FAGI), so it came out $327 short."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started from federal taxable income after subtracting a ~$28,000 federal standard deduction, but Ohio starts from federal AGI and has no standard deduction. It then applied the old 0.501%–5.75% bracket schedule instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It also left out the $1,900 exemptions, the medical deduction, and the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Gave a rounded $1,000 with no computation. Applying the 2026 schedule (about $332 plus 2.75% over $26,050) to $90,944.70 of taxable income gives $2,116.61, and the $200 retirement credit brings it to $1,916.61. The $1,000 answer drops roughly half of that tax."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used $1,850 exemptions instead of $1,900 and skipped the $180.60 unreimbursed medical deduction. It padded its $1,792 figure with an arbitrary 'modest amount' rather than the schedule's roughly $332 base. It also said no nonrefundable credits apply, missing the $200 retirement income credit for the $32,200 pension."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Computed $1,790 from 2.75% over $26,050 without the 2026 base amount of about $332. It said the retirement and joint filing credits reduce the tax, then reported $2,426, which is higher than its own pre-credit figure. The joint filing credit does not apply because the head has no qualifying income, and the $180.60 medical deduction was skipped."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used $2,500 exemptions instead of the $1,900 for MAGI above $80,000, skipped the $180.60 unreimbursed medical deduction, and used the 2024 $360 base instead of about $332. Its stated parts ($360 + 2.75% × $63,875 = $2,116.56) don't add up to the $1,866 it reported. It never subtracted the $200 retirement income credit explicitly."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Explicitly assumed the 2026 schedule has no base amount. In fact tax above $26,050 is about $332 plus 2.75% of the excess. It also skipped the $180.60 unreimbursed medical deduction, which lowers Ohio AGI to $94,744.70, so it came out $327 short."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Stated that Ohio has no standalone deduction for health premiums. Ohio's unreimbursed medical care deduction (R.C. 5747.01(A)(10)) allows the $6,500 premiums plus $800 of other expenses above 7.5% of FAGI, which is $180.60 here. It also used $2,400 exemptions instead of $1,900 above $80,000 MAGI and left out the roughly $332 schedule base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"Used $2,350 exemptions instead of $1,900 and left out the roughly $332 base amount. It then subtracted about $500 of 'retirement income and senior credits'. The retirement income credit is only $200, and the senior citizen credit requires age 65, which neither spouse has reached (61 and 57)."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Correctly used $3,800 of exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used $1,000 exemptions instead of $1,900 and made up a 2.5%/3.5% bracket schedule instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It left out the $200 retirement income credit for the $32,200 pension and the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied the pre-2024 multi-bracket schedule (3.226%, 3.688%) instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It allowed only one $2,400 exemption instead of two $1,900 exemptions. It also said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used $2,800 exemptions instead of the $1,900 for MAGI above $80,000 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied an outdated 3.5% rate above $46,100 instead of the 2026 schedule (about $332 plus 2.75% over $26,050) and used $2,400 exemptions instead of $1,900. It applied the 5% joint filing credit even though the head has no qualifying income. It also missed the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 employer-sponsored insurance premiums from wages, cutting FAGI to $73,717. Federal AGI is actually $94,925.30, so this inflated the 7.5%-floor medical deduction to $1,821 instead of $180.60. It also counted the $50 of over-the-counter expenses, used $2,400 exemptions instead of $1,900, and left out the roughly $332 base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave no derivation. The correct figure is $2,116.61: about $332 plus 2.75% of $90,944.70 over $26,050. Subtracting the $200 retirement credit gives $1,916.61. The $1,695 answer leaves out the schedule's base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The $1,762 answer equals 2.75% × ($90,125 − $26,050). That means it used $2,400 exemptions instead of $1,900, left out the roughly $332 base amount, skipped the $180.60 medical deduction, and never subtracted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used $2,700 exemptions instead of the $1,900 for MAGI above $80,000, applied 2.75% without the roughly $332 base amount, and skipped the $180.60 medical deduction. It also never subtracted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no derivation. The correct figure is $1,916.61: Ohio taxable income of $90,944.70 is taxed at about $332 plus 2.75% over $26,050 ($2,116.61), less the $200 retirement credit. The $1,280.50 answer leaves out the base amount and understates the 2.75% portion."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Correctly used $3,800 of exemptions and the $200 retirement credit. However, it applied 2.75% only to income over $26,050 and left out the roughly $332 base amount. It also skipped the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $1,694.06 answer equals 2.75% × ($94,925 − $26,050) minus $200. That means it subtracted neither the two $1,900 personal exemptions nor the $180.60 medical deduction, and it left out the roughly $332 base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Wrongly said Ohio personal exemptions are $0 above $80,000 MAGI; they are $1,900 each. It also skipped the $180.60 medical deduction and the roughly $332 base amount, and never applied the $200 retirement income credit. These errors partly cancel, which is why the answer lands near the reference."
+us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Subtracted a $30,000 federal standard deduction, which Ohio does not allow, and used $2,400 exemptions instead of $1,900. It then applied 2.75% to all taxable income, ignoring the $26,050 zero bracket and the roughly $332 base amount. It also missed the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its reasoning uses an obsolete 2.85%–4.396% bracket schedule and a joint filing credit that doesn't apply here, and arrives at $3,245. It then reported $1,260.80, which contradicts that work. Neither number follows the 2026 schedule (about $332 plus 2.75% over $26,050), the $1,900 exemptions, the medical deduction, or the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Treated the household as low-income with no tax. Ohio taxable income is $90,944.70, far above the $26,050 zero bracket, which gives $2,116.61 of tax and $1,916.61 after the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Applied a 'standard deduction', but Ohio has none. It starts from federal AGI, subtracts the $180.60 medical deduction and two $1,900 exemptions, and taxes the result at about $332 plus 2.75% over $26,050, less the $200 retirement credit. The $1,340 answer understates that result."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used the 2024 base amount of $360.69 instead of the 2026 schedule's roughly $332 for income over $26,050. It also added the $50 of over-the-counter expenses to the medical deduction ($230.63 instead of $180.60). Together these overstate the tax by about $27."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used $2,400 exemptions instead of the $1,900 for MAGI above $80,000 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $2,150.25 tax on $91,125 uses the 2024 base of $360.69 instead of the 2026 base of about $332. It also left out the $180.60 unreimbursed medical deduction, which lowers Ohio AGI to $94,744.70."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Excluded the $32,200 private pension from Ohio income, but Ohio fully taxes private pension income. Ohio's only pension relief is the nonrefundable retirement income credit, which is $200 for $8,000 or more of retirement income, not $25. It also applied a joint filing credit even though the head has no qualifying income."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Put the $21,208 employer-sponsored insurance premiums into the unreimbursed medical deduction, which produced a $21,388.63 deduction. Only the $6,500 out-of-pocket premiums and the $800 of other expenses count, so the deduction is $180.60. It also used $2,150 exemptions instead of $1,900."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Applied 2.75% only to income above the zero bracket and left out the 2026 schedule's base amount of about $332. It also skipped the $180.60 unreimbursed medical deduction, landing at $1,590 instead of $1,916.61."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Correctly used two $1,900 exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount. It also skipped the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Deducted the full $6,500 of premiums with no floor. Because the head can get employer-paid coverage, those premiums go on line 3 of Ohio's unreimbursed medical care worksheet with the $800 of other expenses. Only the amount above 7.5% of FAGI is deductible, which is $180.60 here."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Said Ohio taxable income drops to zero, but $94,744.70 of Ohio AGI less $3,800 of exemptions leaves $90,944.70. That amount is taxed at about $332 plus 2.75% over $26,050, giving $1,916.61 after the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Used $4,800 of exemptions instead of $3,800 (two $1,900 exemptions above $80,000 MAGI) and left out the roughly $332 base amount. It said there were no nonrefundable credits, missing the $200 retirement income credit, and skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"Applied a 5% joint filing credit even though the head has no qualifying income, so the credit is unavailable. It also left out the roughly $332 base amount and the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got the $1,900 exemptions, the $200 retirement credit, and the joint filing credit's unavailability right. However, it applied 2.75% only above the $26,050 zero bracket and left out the roughly $332 base amount. It also skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Used an outdated 3.5% rate above $46,100 instead of the 2026 schedule (about $332 plus 2.75% over $26,050) and $2,650 exemptions instead of $1,900. It also missed the $200 retirement income credit and the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used $4,800 of exemptions instead of $3,800 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit for the $32,200 pension, and skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Used $2,500 exemptions instead of the $1,900 for MAGI above $80,000, applied 2.75% without the roughly $332 base amount, and skipped the $180.60 medical deduction. It also never subtracted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly used $1,900 exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"Applied senior or aged adjustments to cut taxable income to near zero, but neither spouse is 65. Ohio taxable income is $90,944.70, which gives $1,916.61 of tax after the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Applied no personal exemptions, though two $1,900 exemptions apply above $80,000 MAGI, and skipped the $180.60 medical deduction. It also left out the roughly $332 base amount and missed the $200 retirement income credit. These errors partly cancel."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Gave no derivation. Correctly, $90,944.70 of Ohio taxable income (after the $180.60 medical deduction and $3,800 of exemptions) is taxed at about $332 plus 2.75% over $26,050, less the $200 retirement credit, which gives $1,916.61. The $2,651 answer implies a higher rate and no retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Made up a $9,300 standard deduction, which Ohio does not have. It taxed the first $26,050 at 2.75% (that bracket is 0%) and used 3.5% above it, and its own sum doesn't add up. It also claimed a senior citizen credit, which requires age 65, for the 61-year-old head."
us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It computed MAGI of $82,301, below the 216% FPL limit (211% plus the 5% disregard), but assigned that band to CHIP. In NC that band is Medicaid coverage for older children (the OLDER_CHILD category), so the child is Medicaid-eligible and therefore excluded from CHIP."
us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It understated household income at $61,093; wages minus traditional 401(k) and IRA contributions give about $82,301. It also labeled the 211% FPL threshold as NC Health Choice/CHIP, but that program was folded into Medicaid. An 11-year-old under that limit is Medicaid-eligible (OLDER_CHILD), which rules out CHIP."
us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It set NC Medicaid for an 11-year-old at 133% FPL and treated NC Health Choice (211% FPL) as a separate CHIP tier. NC now covers children aged 6-18 up to 211% FPL through Medicaid (OLDER_CHILD), so this child is Medicaid-eligible and not CHIP-eligible."
us,scenario_026,child1_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed the child above a lower Medicaid limit and under a separate NC Health Choice CHIP limit of 211% FPL. It missed that NC's 211% coverage for school-age children is Medicaid (OLDER_CHILD), and Medicaid eligibility excludes the child from CHIP."
us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child1_chip_eligible. The correct derivation finds the 11-year-old Medicaid-eligible under NC's OLDER_CHILD category, which makes CHIP eligibility 0."
-us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Used a 133% FPL limit (138% with the disregard) for NC children ages 6–18. That is the pre-2023 limit, from before NC Health Choice merged into Medicaid and raised the limit to 211% FPL (216% with the disregard). The model's own MAGI of about $82,300 is 2.13x FPL, which falls under that limit."
-us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the income ratio right (~213% FPL) but compared it to the outdated 133% FPL limit for NC children ages 6–18. NC Medicaid now covers this group up to 211% FPL (216% with the 5% disregard), so 2.13x FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Used a poverty line of $27,465 for five people, far below the 2026 guideline of about $38,640. It also applied a made-up 200% limit and compared gross wages instead of MAGI net of 401(k)/IRA contributions. It then treated employer-sponsored insurance as disqualifying, but Medicaid has no uninsured requirement. MAGI of $82,301 is 2.13x FPL, under NC's 216% effective child limit."
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied a 138% FPL limit (~$51,957) to an 11-year-old. After the 2023 merger of NC Health Choice into Medicaid, NC's limit for ages 6–18 is 211% FPL (216% with the disregard). MAGI of $82,301 (2.13x FPL) is under that limit."
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Used a 138% FPL limit and got its dollar value wrong (~$45,000). It also compared gross wages instead of MAGI. NC's Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies."
-us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the correct 211% FPL limit for ages 6–18 but put income at ~260% FPL. MAGI is only the spouse's $85,209 in wages minus $2,778 traditional 401(k) and $130 traditional IRA contributions, or $82,301. That is 2.13x FPL, under the 211%/216% limit."
-us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Put income at ~219% FPL by dividing MAGI by the 2025 poverty guideline ($37,650) instead of the 2026 guideline. It also left out the 5% MAGI disregard. Under 2026 parameters, MAGI of $82,301 is 2.13x FPL, below NC's 211% child limit (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Only CHIP requires a child to be uninsured; Medicaid does not. Child 1 meets the OLDER_CHILD income test with MAGI at 2.13x FPL, under NC's 211% FPL limit (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Put income at ~$93,000 by adding the $7,746 overtime premium on top of wages that already include it, and did not subtract the traditional 401(k)/IRA contributions. It also priced 211% FPL at only ~$67,000 instead of about $81,500 (or ~$83,500 with the disregard). Correct MAGI is $82,301 (2.13x FPL), which is eligible."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Compared MAGI of about $82k to a 133% FPL limit. That limit is obsolete: NC Medicaid covers children ages 6–18 up to 211% FPL (216% with the disregard), and $82,301 is 2.13x FPL."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put income at ~$93k by adding the overtime premium on top of wages that already include it, and ignored the pre-tax 401(k)/IRA deductions. It also priced the 210% limit at only ~$77k and left out the 5% disregard. Correct MAGI is $82,301 (2.13x FPL), below the 216% effective limit."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but compared it to 133% FPL ($51,205). That is the pre-2023 NC Medicaid limit for ages 6–18; the current limit is 211% FPL (216% with the disregard), so 2.13x FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but still found it over the child limit, which means it used a threshold below 2.13x FPL. NC Medicaid covers children ages 6–18 up to 211% FPL (216% with the 5% disregard)."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Compared gross wages of $85,209 to a bare 210% limit. It skipped the traditional 401(k) and IRA deductions that bring MAGI down to $82,301, and it left out the 5% disregard that raises the effective limit to 216%. At 2.13x FPL, Child 1 qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied a made-up 147% FPL limit for ages 6–18. It also subtracted the head's $21,208 ESI premium from MAGI, getting $61,093. NC's limit is 211% FPL (216% with the disregard), so even the correct MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was over the limit without giving a threshold. MAGI is $82,301 (2.13x FPL), which is under NC's 211% FPL Medicaid limit for ages 6–18 (216% with the disregard). A 'No' only follows from the pre-2023 133% limit."
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied a 133% FPL Medicaid limit and sent the child to CHIP instead. NC Health Choice was folded into Medicaid in 2023, so Medicaid itself now covers children ages 6–18 up to 211% FPL (216% with the disregard). MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Compared income to the obsolete 133% FPL limit, and understated MAGI at $61,093 by subtracting ESI premiums. NC's child Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under it."
-us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Said Child 1 was ineligible without showing any income test. MAGI is $82,301 (2.13x FPL), under NC's 211% FPL limit for ages 6–18 (216% with the disregard). A 'No' matches the outdated 133% limit."
-us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income was over the NC limit without naming it. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for children ages 6–18 (216% with the disregard), so the model used an outdated lower threshold."
-us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income was over the limit without naming a threshold. Correct MAGI is $82,301 (2.13x FPL), inside NC's 211% FPL OLDER_CHILD limit (216% with the disregard). A 'No' reflects the pre-2023 133% FPL limit."
-us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said income was over NC's child limit. Since NC Health Choice merged into Medicaid, the limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) falls under it."
-us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Only CHIP has an uninsured requirement; Medicaid does not. Child 1 qualifies in the OLDER_CHILD category with MAGI at 2.13x FPL, under NC's 211% limit (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used the correct 211% limit but tested gross wages of $85,209. It skipped the $2,778 traditional 401(k) and $130 traditional IRA deductions that bring MAGI to $82,301, and it left out the 5% disregard (216% effective limit). At 2.13x FPL, Child 1 is eligible."
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings were over the Medicaid threshold without naming it. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for school-age children (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Ruled Child 1 ineligible based on income level alone, with no threshold. MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL Medicaid limit for ages 6–18 (216% with the disregard), so the OLDER_CHILD income test is met."
-us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income was over the school-age child limit for five people. NC's Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used an estimated NC child threshold below the real one. The actual limit for ages 6–18 is 211% FPL (216% with the 5% disregard), and MAGI of $82,301 (2.13x FPL) falls under it."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's limit for an 11-year-old. That limit is 211% FPL (216% with the disregard) since NC Health Choice merged into Medicaid, and MAGI of $82,301 is 2.13x FPL, below it."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income was over NC's child Medicaid limit. MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL limit for ages 6–18 (216% with the disregard), so the model used a threshold that is too low."
-us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said income exceeded NC's age-specific limit. For ages 6–18 that limit is 211% FPL (216% with the disregard), and household MAGI of $82,301 is 2.13x FPL, which qualifies."
-us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeded the child Medicaid limit. NC covers children ages 6–18 in Medicaid up to 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under that."
-us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but still found it over NC's child limit for five people, which means it used a threshold below 2.13x FPL. The OLDER_CHILD limit is 211% FPL (216% with the 5% disregard)."
-us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"Said income was over the limit and treated employer-sponsored insurance as disqualifying. Medicaid has no uninsured requirement, and MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL child limit (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,Applied a 133% FPL limit to children ages 6–18. That limit ended when NC Health Choice merged into Medicaid in 2023; the current limit is 211% FPL (216% with the disregard). MAGI of 2.13x FPL qualifies.
-us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Applied the obsolete 133% FPL (~$51,000) limit for ages 6–18, and understated MAGI at $74,555. NC's current limit is 211% FPL (216% with the disregard), so even the correct MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Said NC Medicaid for ages 6–18 stops at about 138% FPL. Since 2023 it covers this group up to 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under that."
-us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Overstated income at ~240% FPL (MAGI is $82,301, or 2.13x FPL) and compared it to the outdated 133% FPL limit. NC's current Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard)."
-us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Capped Medicaid for ages 6–18 at 133% FPL. That is outdated: NC Health Choice's former 211% FPL range is now part of Medicaid (216% with the disregard). MAGI of $82,301 (2.13x FPL) qualifies, and Medicaid, unlike CHIP, does not require the child to be uninsured."
-us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for child1_medicaid_eligible. The expected answer is 1: MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL Medicaid limit for ages 6–18."
-us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (138% with the disregard) for ages 6–18 and overstated income at ~226% FPL. NC's current limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies."
-us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income was well above NC's Medicaid for Children limit. MAGI of $82,301 is 2.13x FPL, under the current 211% FPL limit for ages 6–18 (216% with the disregard), so the model used a limit that is too low."
-us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"Put children ages 6–18 under a 133% FPL limit. NC Medicaid covers this group up to 211% FPL (216% with the disregard), so even the model's own ~198% FPL estimate qualifies, as does the correct 2.13x FPL."
-us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Correctly found income under the child limit, then ruled Child 1 out because of employer-sponsored insurance. Medicaid has no uninsured requirement (only CHIP does), so the OLDER_CHILD income test alone makes Child 1 eligible."
-us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Said income was too high without naming a threshold. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for ages 6–18 (216% with the 5% disregard)."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Used the old 133%/138% FPL limit for children ages 6–18. Since NC Health Choice merged into Medicaid, NC covers these children up to 211% FPL (216% with the disregard). Its MAGI of about $82,300 is 213% of the 2026 FPL for five, which is under that limit."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the income ratio right at about 213% FPL but compared it to a stale 133% FPL limit for children 6–18. NC's older-child Medicaid limit is 211% FPL (216% with the 5-point disregard), so Child 1 is eligible."
+us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a 200% limit and an understated FPL of $27,465 for five people; the 2026 figure is about $38,640. It also compared gross wages instead of MAGI ($82,301) and treated ESI as disqualifying, though Medicaid has no uninsured requirement. MAGI is 213% FPL, within NC's 211%+5% older-child limit."
+us,scenario_026,child1_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said income was 'well above' the child Medicaid limit, which is consistent with the old 133–138% FPL standard. NC's older-child Medicaid category runs to 211% FPL (216% with the disregard), and the household's MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a 138% FPL (~$51,957) limit for NC children 6–18. After the Health Choice merger, NC Medicaid covers this group up to 211% FPL plus the 5-point disregard. MAGI of $82,301 is 213% FPL and qualifies."
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Applied a 138% FPL limit, and miscalculated it at about $45,000 for a family of five. NC's actual older-child Medicaid limit is 211% FPL (216% with the disregard), which covers MAGI at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Had the right 211% FPL limit but overstated income at about 260% FPL. MAGI is $85,209 in wages, which already include overtime, minus the $2,778 traditional 401(k) and $130 IRA deductions, or $82,301. That is 213% FPL, within 211% plus the 5-point disregard."
+us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Put income at 219% FPL, which matches dividing by the 2025 guideline ($37,650) instead of the uprated 2026 FPL. That figure gives 213%. It also left out the 5-point MAGI disregard that raises NC's 211% older-child limit to 216%."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no uninsured requirement; that bar applies only to CHIP. On income, MAGI of 213% FPL is within NC's older-child Medicaid limit."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime premium on top of $85,209 in wages, even though wages already include overtime. It also left out the pre-tax 401(k) and IRA deductions and turned 211% FPL into about $67,000 using a wrong FPL. Correct MAGI is $82,301, or 213% of the 2026 FPL, which is within NC's 211%+5% older-child limit."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children 6–18. NC Medicaid now covers that age group up to 211% FPL (216% with the disregard), and the household's MAGI of about $82,300 is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put household income at about $93k by adding the overtime premium to wages that already include overtime, and skipped the 401(k) and IRA deductions. Correct MAGI is $82,301, or 213% FPL, which is within NC's 211% older-child limit plus the 5-point disregard."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but tested it against a 133% FPL limit. NC's Medicaid limit for children 6–18 is 211% FPL (216% with the disregard), so 213% FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but said it exceeded the child Medicaid limit. That $82,301 is 213% of the 2026 FPL for five, which falls within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Compared gross wages ($85,209) to a 210% limit. It should have used MAGI after the 401(k) and IRA deductions ($82,301, or 213% FPL) and included the 5-point disregard, which brings NC's older-child Medicaid limit to 216%."
+us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 147% FPL limit for children 6–18 in place of NC's 211% (216% with the disregard) older-child Medicaid limit. It also wrongly subtracted the head's ESI premiums from MAGI. Correct MAGI of $82,301 is 213% FPL and qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was above the threshold without naming one, which is consistent with the old 133–138% FPL child limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Used a 133% FPL Medicaid limit and sent Child 1 to CHIP instead. NC merged Health Choice into Medicaid, so children 6–18 up to 211% FPL (216% with the disregard) are in Medicaid, and MAGI of $82,301 (213% FPL) qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children. NC's older-child Medicaid limit is 211% FPL (216% with the disregard). It also wrongly took the head's ESI premiums out of MAGI, but even correct MAGI of 213% FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Said Child 1 was not eligible without giving a reason, which is consistent with applying the old 133–138% FPL child limit. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's limit without giving one, which is consistent with the old 133–138% FPL standard. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, which covers MAGI at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income exceeded eligibility limits without a figure, which is consistent with the old 133–138% FPL child threshold. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), above the household's 213% FPL MAGI."
+us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's child Medicaid limits without naming one, which is consistent with the old 133–138% FPL threshold. After the Health Choice merger, NC covers children 6–18 up to 211% FPL plus the disregard, and MAGI of 213% FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no uninsured requirement; only CHIP bars children with ESI. On income, MAGI of 213% FPL is within NC's older-child limit."
+us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"Had the right 211% FPL limit but compared gross wages ($85,209) instead of MAGI ($82,301 after the 401(k) and IRA deductions). It also left out the 5-point disregard. MAGI of 213% FPL falls within the 216% effective limit."
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings were above the child Medicaid threshold, which is consistent with the old 133–138% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Ruled out Medicaid from the wage level alone without testing the right threshold. MAGI of $82,301 is 213% of the 2026 FPL for five, which falls within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard."
+us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income exceeded the school-age child Medicaid limit, which is consistent with the old 133–138% FPL standard. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and the household is at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Estimated NC's child Medicaid threshold too low; this answer is consistent with the old 133–138% FPL limit. The older-child limit is 211% FPL plus the 5-point disregard, which covers MAGI at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's limit for an 11-year-old. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 213% of the 2026 FPL for five, so Child 1 is eligible."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income was above NC's child Medicaid limit, which is consistent with the old 133–138% FPL standard. The current older-child limit is 211% FPL plus the 5-point disregard, above the household's 213% FPL MAGI."
+us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's Medicaid limit for age 11. NC's older-child Medicaid category runs to 211% FPL (216% with the disregard), which covers MAGI of $82,301 at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeded the child Medicaid limit, which is consistent with the old 133–138% FPL threshold. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but said it exceeded NC's limit for five people. That MAGI is 213% of the 2026 FPL, within NC's older-child Medicaid limit of 211% FPL (216% with the disregard)."
+us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"Said income exceeded NC's Medicaid threshold and cited ESI as an added bar. NC's older-child Medicaid limit is 211% FPL plus the disregard, which covers MAGI at 213% FPL. ESI does not disqualify a child from Medicaid; only CHIP has that bar."
+us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children 6–18. NC Medicaid covers this group up to 211% FPL (216% with the disregard), and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Used a 133% FPL (~$51,000) limit for children 6–18 in place of NC's 211% (216% with the disregard) older-child Medicaid limit. It also understated MAGI at $74,555; the correct $82,301 is 213% FPL and qualifies."
+us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Used a 138% FPL limit for children 6–18. After the Health Choice merger, NC Medicaid covers this group up to 211% FPL plus the 5-point disregard, which includes MAGI at 213% FPL."
+us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Used a 133% FPL limit and overstated income at about 240% FPL. Correct MAGI of $82,301 is 213% of the 2026 FPL, within NC's older-child Medicaid limit of 211% FPL plus the disregard."
+us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Used a 133% FPL (~$50,000) limit for children 6–18. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), so the household's 213% FPL MAGI qualifies. The CHIP uninsured rule it cited does not apply to Medicaid."
+us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for child1_medicaid_eligible, so nothing could be compared with the eligible result: MAGI at 213% FPL under NC's older-child limit."
+us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Used a 133% (138% with disregard) FPL limit for children 6–18. NC's actual older-child Medicaid limit is 211% FPL (216% with disregard). Its income estimate of about 226% FPL also overstated MAGI, which is 213% of the 2026 FPL."
+us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income was 'well above' NC's children's Medicaid limits, which is consistent with the old 133–138% FPL standard. NC Medicaid covers children 6–18 up to 211% FPL plus the disregard, and MAGI is 213% FPL."
+us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,Put Child 1 in the 6–18 group but used a 133% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard). Even its own 198% FPL estimate would have qualified under that limit.
+us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"Found income within the child Medicaid limit, then wrongly treated Child 1's employer-sponsored insurance as disqualifying. Medicaid has no uninsured requirement; only CHIP bars children with ESI."
+us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Said income was too high for child Medicaid without a threshold, which is consistent with the old 133–138% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 213% FPL."
us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It compared MAGI of $82,301 to a 216% FPL limit and labeled that the NC CHIP limit. That band is the NC Medicaid limit for children aged 6-18. The child qualifies for Medicaid under the older-child category, and Medicaid eligibility excludes CHIP, so the answer is 0, not 1."
us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It found income of $61,093 below 211% FPL and treated that as NC Health Choice (CHIP) eligibility. At that income an 11-year-old in NC is Medicaid-eligible under the older-child category, since NC Health Choice is now part of Medicaid. A Medicaid-eligible child is not CHIP-eligible."
us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It used the federal minimum of 133% FPL as the Medicaid ceiling for school-age children and put the 133-211% band in CHIP. NC Medicaid's older-child category covers an 11-year-old across that band, so at MAGI of $74,555 the child is Medicaid-eligible. Medicaid eligibility excludes CHIP."
us,scenario_026,child2_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed household MAGI in a CHIP band above Medicaid and at or below 211% FPL, assuming NC Medicaid for an 11-year-old stops at a lower limit. The engine's NC Medicaid older-child category covers this child at this income. Because the child is Medicaid-eligible, the child is not CHIP-eligible."
us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_chip_eligible, so there is no substantive answer to grade. The correct answer is 0: the child is Medicaid-eligible under NC's older-child category, which excludes CHIP."
-us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model got MAGI right at about $82,301 but divided by a 2025-level FPL (about $37,600), which gave 219% instead of the engine's 2.13× the 2026 FPL. It then compared that against a bare 211% limit without the 5-point MAGI disregard, even though NC's older-child Medicaid threshold covers 213% FPL."
-us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model correctly put income at about 213% FPL but used the pre-2023 133% Medicaid limit for ages 6-18. NC merged Health Choice into Medicaid, raising the older-child limit to 211% FPL (about 216% with the disregard), so 213% qualifies."
-us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model said the household has two earners, but Head has no wages; the spouse's wages are the only income. It also treated employer-sponsored coverage as a bar to Medicaid, which it is not. MAGI of about $82,300 is 2.13× FPL, within NC's older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model first used a 211% limit with an understated FPL of $36,580, then settled on a 133% FPL limit for ages 6-18. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and the household's 2.13× FPL MAGI falls under it."
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model split coverage into 138% Medicaid and 211% CHIP, the structure NC dropped when Health Choice moved into Medicaid. It also used gross wages of about $85,000 instead of MAGI net of the 401(k) and IRA deductions, and an understated FPL of $36,580. Correct MAGI is 2.13× FPL, within NC's older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated household income at about 260% FPL. The only income is the spouse's wages net of the traditional 401(k) and IRA deductions (MAGI about $82,300), which is 2.13× the 2026 FPL for five and within NC's older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model put income at 219% FPL using a too-low FPL, when the engine's figure against the 2026 guideline is 2.13×. It then treated that as over the child limit, ignoring that NC's post-merger older-child Medicaid limit (211% plus the 5-point disregard) covers 213%."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model ruled Child 2 out because the child has employer-sponsored insurance, but Medicaid has no uninsured requirement and ESI does not disqualify a child. It never ran the income test: MAGI of 2.13× FPL is within NC's older-child limit."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model named the right roughly 216% FPL limit but turned it into about $67,000 using a far-too-low FPL. 216% of the 2026 FPL for five is about $83,500, above MAGI of about $82,300 (which excludes the 401(k) and IRA deductions, not adds overtime on top), so the child is eligible."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL Medicaid limit for school-age children. Since the Health Choice merger, NC Medicaid covers ages 6-18 up to 211% FPL plus the disregard, so the household's roughly $82,000 MAGI (2.13× FPL) qualifies."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,The model applied a flat 210% FPL limit without the 5-point MAGI disregard. NC's older-child Medicaid threshold in PolicyEngine covers the household's 2.13× FPL MAGI.
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but tested it against a 133% FPL limit. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and $82,301 is 2.13× the 2026 FPL for five, which qualifies."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but said it exceeded NC's child limit. That MAGI is 2.13× the 2026 FPL for five, within the older-child Medicaid limit of 211% plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model compared gross wages of $85,209 against a bare 210% FPL limit. It skipped the 401(k) and IRA deductions that bring MAGI to about $82,300 (2.13× FPL) and left out the 5-point disregard that puts NC's older-child limit above that level."
-us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit for ages 6-18, and it also subtracted ESI premiums nobody pays through payroll, getting a MAGI of $61,093 instead of about $82,300. NC's older-child Medicaid limit covers the correct MAGI of 2.13× FPL."
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income was over the threshold without citing a limit. The correct MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the 5-point disregard. Its answer fits the outdated 133% Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but applied a 133% Medicaid limit and sent the child to CHIP. NC ended separate Health Choice CHIP in 2023 and now covers ages 6-18 in Medicaid up to 211% FPL plus the disregard, which includes 2.13× FPL."
-us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model subtracted insurance premiums that are not pre-tax, getting $61,093 instead of MAGI of about $82,300, and tested against the obsolete 133% FPL limit. NC's older-child Medicaid limit of 211% plus the disregard covers the household's 2.13× FPL."
-us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. MAGI (spouse wages minus the 401(k) and IRA deductions, about $82,300) is 2.13× FPL, within NC's older-child Medicaid limit, so answering 'not eligible' means it applied a lower child threshold such as the old 133%."
-us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeded the limit without citing one. MAGI of 2.13× FPL is under NC's post-Health Choice older-child limit of 211% FPL plus the 5-point disregard, so it used an outdated or too-low threshold."
-us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model said income exceeded the limit without citing one. Household MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the disregard, so it applied a threshold that is too low."
-us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model said income exceeded NC's child limits without citing a number. NC Medicaid covers ages 6-18 up to 211% FPL plus the 5-point disregard, and the household's MAGI is 2.13× FPL, so the child qualifies."
-us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model got income right at about 213% FPL but used the pre-2023 split of 133% Medicaid and 210% Health Choice CHIP, and it also counted ESI against eligibility. NC now covers ages 6-18 in Medicaid up to 211% FPL plus the disregard (about 216%), and ESI does not disqualify."
-us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"The model compared gross wages of $85,209 to a bare 211% FPL. It skipped the 401(k) and IRA deductions that bring MAGI to 2.13× FPL and left out the 5-point MAGI disregard that puts NC's older-child limit above that level."
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said earnings were above the Medicaid threshold without computing a ratio. Household MAGI is 2.13× FPL, under NC's older-child Medicaid limit (211% FPL plus the 5-point disregard)."
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model denied eligibility based on a vague sense that income was too high, without an FPL comparison. MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model said income of about $85,209 was 'well above' NC's child limit. After the 401(k) and IRA deductions, MAGI is 2.13× FPL, and NC's older-child Medicaid limit (211% plus the 5-point disregard) covers that."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an 'estimated' NC child threshold set too low. The actual older-child Medicaid limit is 211% FPL plus the 5-point disregard, which covers the household's 2.13× FPL MAGI."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeded NC's limit for an 11-year-old. MAGI is 2.13× FPL, and NC's post-2023 older-child Medicaid limit of 211% plus the disregard includes that."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model said income was above the NC child limit without quantifying it. Household MAGI is 2.13× FPL, below NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied a child income limit below 213% FPL. Since the 2023 Health Choice merger, NC Medicaid for ages 6-18 extends to 211% FPL plus the disregard, which covers the household's 2.13× FPL MAGI."
-us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeded the child Medicaid limit. Household MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but set NC's child limit too low. $82,301 is 2.13× the 2026 FPL for five, within the older-child Medicaid limit of 211% plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model said income was over the NC threshold and counted ESI against eligibility. ESI does not bar Medicaid, and MAGI of 2.13× FPL is within NC's older-child limit (211% plus the disregard)."
-us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for children 6-18, which NC abandoned when Health Choice moved into Medicaid. The current limit of 211% plus the disregard covers the household's 2.13× FPL MAGI."
-us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model understated MAGI at $74,555 instead of about $82,300 and tested it against the obsolete 133% FPL limit. NC's older-child Medicaid limit of 211% plus the 5-point disregard covers the household's actual 2.13× FPL."
-us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model said MAGI was above NC's limit for ages 6-18. MAGI is 2.13× FPL, and NC's post-merger older-child Medicaid limit (211% FPL plus the disregard) includes it."
-us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model overstated income at about 240% FPL instead of 2.13× and used the obsolete 133% child limit. NC's older-child Medicaid limit of 211% plus the disregard covers the correct ratio.
-us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit for ages 6-18 and treated ESI as disqualifying. NC Medicaid covers these children up to 211% FPL plus the disregard regardless of ESI, and household MAGI is 2.13× FPL."
-us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for child2_medicaid_eligible, so there was no usable answer to score."
-us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133% FPL limit (138% with the disregard) for ages 6-18 and overstated income at about 226% FPL. The correct MAGI is 2.13× FPL, within NC's post-merger older-child limit of 211% plus the disregard."
-us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model said income was 'well above' NC's children's limit without computing it. Household MAGI is 2.13× FPL, within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed 11-year-olds in a 133% FPL Medicaid group, the limit NC replaced with 211% FPL plus the disregard when Health Choice moved into Medicaid. Its own estimate of about 198% FPL, and the engine's 2.13×, both fall under the actual limit."
-us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit and an understated size-5 FPL of $33,700, and it counted ESI against eligibility. NC's older-child Medicaid limit is 211% FPL plus the disregard, and household MAGI is 2.13× the 2026 FPL, so the child qualifies."
-us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model said income was too high without a threshold comparison. MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the 5-point disregard."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but divided by a stale 5-person FPL of about $37,600–$37,900 and came out at 219%, when the 2026 FPL of about $38,700 gives 213%. It also compared against 211% without adding the 5-point MAGI disregard, which raises NC's effective limit for ages 6–18 to 216%."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It correctly put income at about 213% FPL but used the outdated 133% Medicaid limit for ages 6–18. NC Health Choice merged into Medicaid in 2023, so Medicaid now covers these children up to 211% FPL (216% with the disregard), and 213% qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It wrongly treated the household as having two earners (the head has no wages) and counted the child's employer-sponsored insurance against Medicaid eligibility. Medicaid has no uninsured requirement, and household MAGI of 2.13 times FPL is within NC's 216% effective limit for children ages 6–18."
+us,scenario_026,child2_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the child limit, but MAGI is 2.13 times the 2026 FPL for five. NC Medicaid covers ages 6–18 up to 211% FPL plus the 5-point disregard (216%), so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2024 FPL for five ($36,580) instead of the 2026 value (about $38,700) and compared against 211% without the 5-point disregard. It then switched to a 133% Medicaid limit for ages 6–18, which NC abandoned when Health Choice merged into Medicaid. Correctly computed, MAGI is 2.13 times FPL, under the 216% limit."
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used gross wages of about $85,000 without subtracting the $2,778 traditional 401(k) and $130 IRA deductions, used the 2024 FPL of $36,580, and treated 138% as NC's Medicaid limit with CHIP above it. NC Medicaid covers children 6–18 up to 211% plus the 5-point disregard, and MAGI of about $82,300 is 2.13 times the 2026 FPL, so the child qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It overstated income at about 260% FPL. MAGI of about $82,300 (wages minus the traditional 401(k) and IRA deductions) is 2.13 times the 2026 FPL for five, which is under NC's 216% effective Medicaid limit for children ages 6–18."
+us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Its figure of 219% FPL comes from dividing by the 2025 FPL for five ($37,650) instead of the 2026 value. MAGI is actually 2.13 times FPL, under NC's 211% limit plus the 5-point disregard (216%) for children ages 6–18."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It made employer-sponsored insurance a disqualifier, but Medicaid has no uninsured requirement. That rule belongs to CHIP. Child 2 qualifies as an older child because household MAGI of 2.13 times FPL is within NC's 216% limit."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It named roughly the right 216% limit but converted it to about $67,000 for five, when 216% of the 2026 FPL (about $38,700) is about $83,500. It also added overtime on top of wages, although the $85,209 already includes overtime. MAGI of about $82,300 is below the correct dollar limit."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used 133% FPL as the child Medicaid limit, but NC Medicaid has covered children 6–18 up to 211% (216% with the disregard) since Health Choice merged into it in 2023. Its $82,000 of income is 2.13 times the 2026 FPL, so the child qualifies."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It compared income against a bare 210% limit without the 5-point MAGI disregard. NC's effective limit for ages 6–18 is 216% FPL, and household MAGI is 2.13 times FPL, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but tested it against 133% FPL. NC Medicaid now covers children 6–18 up to 211% FPL plus the 5-point disregard (about $83,500 for five in 2026), so $82,301 qualifies."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but wrongly concluded that this exceeds NC's child limit. That income is 2.13 times the 2026 FPL for five, under the 216% effective older-child limit (211% plus the 5-point disregard)."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It tested gross wages of $85,209 instead of MAGI net of the traditional 401(k) and IRA deductions (about $82,300), and it used 210% without the 5-point disregard. Correct MAGI is 2.13 times FPL, under the 216% limit."
+us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used 133% FPL as the Medicaid limit for ages 6–18, but NC Medicaid covers these children up to 211% FPL plus the 5-point disregard after the Health Choice merger. It also wrongly subtracted insurance premiums to get $61,093, when MAGI is about $82,300 (2.13 times FPL), which is still within the limit."
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,It asserted that income is above the threshold without applying NC's actual limit. Household MAGI of 2.13 times FPL is within NC Medicaid's 216% effective limit (211% plus the 5-point disregard) for children ages 6–18.
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but applied the pre-2023 133% FPL Medicaid limit and sent the child to CHIP. Since NC Health Choice merged into Medicaid, older children are Medicaid-eligible up to 211% FPL plus the 5-point disregard, which covers 2.13 times FPL."
+us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a 133% FPL limit for children, but NC Medicaid covers ages 6–18 up to 211% FPL plus the 5-point disregard. Its income figure was also wrong: $61,093 rather than about $82,300 MAGI (2.13 times FPL), which still qualifies."
+us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. MAGI of about $82,300 is 2.13 times the 2026 FPL for five, within NC Medicaid's 216% effective limit for children ages 6–18, so the answer should be eligible. Its 'not eligible' is consistent with applying an outdated 133% limit."
+us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's limit without naming one. MAGI of 2.13 times FPL is under NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard (216%), so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,It said income exceeds the limit without applying NC's actual threshold. Household MAGI of 2.13 times FPL falls within the 216% effective Medicaid limit for children ages 6–18.
+us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's child Medicaid limits, but NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard. The household's MAGI is 2.13 times FPL, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It estimated income at about 213% FPL but used a 133% Medicaid limit with separate CHIP up to 210%, a structure that ended when Health Choice merged into Medicaid in 2023. It also left out the 5-point disregard, which makes the effective Medicaid limit 216%. It wrongly cited the child's employer-sponsored insurance as a reason against eligibility."
+us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"It compared gross wages of $85,209 against 211% FPL. Correct MAGI subtracts the traditional 401(k) and IRA deductions (about $82,300, or 2.13 times FPL), and the limit gets a 5-point disregard to 216%, so the child qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,It said earnings exceed the Medicaid threshold without computing one. Household MAGI of 2.13 times FPL is within NC's 216% effective Medicaid limit for school-age children.
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed income was disqualifying without comparing MAGI to NC's limit. MAGI of about $82,300 is 2.13 times FPL, under the 211%-plus-5-point limit for children ages 6–18, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It treated $85,209 as 'well above' NC's child limit. MAGI net of the traditional 401(k) and IRA deductions is about $82,300, or 2.13 times the 2026 FPL for five, which is within the 216% effective limit for ages 6–18."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its estimated NC threshold was too low. NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard, and household MAGI is 2.13 times FPL, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It claimed MAGI exceeds NC's limit for an 11-year-old. MAGI is 2.13 times the 2026 FPL, under the 216% effective older-child limit (211% plus the 5-point disregard)."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was above NC's child limit, but NC Medicaid covers ages 6–18 up to 216% FPL after the disregard. Household MAGI is 2.13 times FPL, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It said income exceeds the age-specific limit, but NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard. Household MAGI of 2.13 times FPL is under it."
+us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It asserted that income exceeds the child limit without applying NC's 216% effective threshold for ages 6–18. Household MAGI is 2.13 times FPL, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but misjudged the limit. That is 2.13 times the 2026 FPL for five, below NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard (about $83,500)."
+us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It cited employer-sponsored insurance as a disqualifier, but Medicaid has no uninsured requirement. It also judged income above the limit when MAGI of 2.13 times FPL is within NC's 216% effective limit for children 6–18."
+us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It applied a 133% FPL Medicaid limit for children 6–18, but NC Medicaid has covered these children up to 211% FPL (216% with the disregard) since Health Choice merged into it. MAGI of 2.13 times FPL qualifies."
+us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL limit for children 6–18 instead of NC's post-merger Medicaid limit of 211% FPL plus the 5-point disregard. Its MAGI was also wrong at $74,555 instead of about $82,300 (2.13 times FPL), which still qualifies."
+us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It claimed MAGI exceeds NC's limit for ages 6–18, but that limit is 211% FPL plus the 5-point disregard. Household MAGI of 2.13 times FPL is under it."
+us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It overstated income at about 240% FPL and used a 133% limit. MAGI is 2.13 times the 2026 FPL, and NC Medicaid covers children 6–18 up to 216% after the disregard, so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It applied a 133% FPL Medicaid limit for ages 6–18, but NC Medicaid covers these children up to 211% FPL plus the 5-point disregard. Household MAGI is 2.13 times FPL, so Child 2 qualifies as an older child regardless of employer-sponsored insurance."
+us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value or explanation for child2_medicaid_eligible, so no answer was recorded."
+us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133% limit (138% with the disregard) for children 6–18, but NC Medicaid covers them up to 211% (216% with the disregard). It also overstated income at 226% FPL when MAGI is 2.13 times the 2026 FPL."
+us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income is 'well above' NC's child Medicaid limit, but household MAGI of 2.13 times FPL is within the 216% effective limit (211% plus the 5-point disregard) for ages 6–18."
+us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed the child in NC's 6–18 group but capped that group at 133% FPL. NC Medicaid covers these children up to 211% FPL plus the 5-point disregard, so even its own 198% estimate, and the actual 2.13 times FPL, qualify."
+us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 133% limit and an understated FPL of $33,700 for five, then compared gross wages to it. NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard against the 2026 FPL of about $38,700, and MAGI of about $82,300 is 2.13 times FPL, which qualifies."
+us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called household income too high without applying NC's threshold. MAGI is 2.13 times FPL, within the 216% effective Medicaid limit for children ages 6–18."
us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133%-211% FPL CHIP band for NC children aged 6-18, which assumes Medicaid for older children stops at 133% FPL. At its own 174% FPL figure, Child 3 falls under North Carolina's OLDER_CHILD Medicaid category. Medicaid eligibility excludes CHIP, so the answer is 0."
us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It checked only whether $82,301 MAGI was under a 216% FPL 'CHIP limit' and never tested Medicaid first. Child 3 is Medicaid-eligible under the NC OLDER_CHILD category, and that Medicaid eligibility bars CHIP no matter where income falls against the CHIP ceiling."
us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It concluded CHIP eligibility because $61,093 is under 211% FPL, treating that ceiling as a CHIP limit rather than checking Medicaid. At that income Child 3 qualifies for NC Medicaid under the OLDER_CHILD category, and Medicaid-eligible children are excluded from CHIP."
@@ -2081,51 +2154,52 @@ us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,F
us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"It treated 133% FPL as the top of Medicaid for a 9-year-old and put its $74,555 MAGI in a 133%-211% CHIP band. North Carolina's OLDER_CHILD Medicaid category covers Child 3 at this income, and Medicaid eligibility excludes CHIP."
us,scenario_026,child3_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"It asserted that household MAGI was 'above Medicaid' and at or below 211% FPL, which puts the child in a separate CHIP tier. In fact Child 3 meets North Carolina's OLDER_CHILD Medicaid category, so the child is Medicaid-eligible and therefore not CHIP-eligible."
us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"It stated outright that Child 3 is 'not Medicaid-eligible', and it measured income against a 6-person FPL even though the household has 5 people. Child 3 is Medicaid-eligible under North Carolina's OLDER_CHILD category, which bars CHIP, so 216% NC Health Choice limit is irrelevant."
-us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found MAGI of about $82,301, which is about 2x FPL. It then used a 138% FPL Medicaid limit for ages 6-18 and moved the 138-211% band into CHIP. That ignores the 2023 merger of NC Health Choice into Medicaid, which puts the older-child Medicaid limit at 211% FPL (216% with the disregard). At 2.13x FPL, Child 3 qualifies."
-us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the income ratio right at about 213% FPL but compared it with a 133% child limit. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the 5% disregard), so 213% is eligible."
-us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It named a 207% FPL limit but converted it to only $56,232 for a family of five, which is about 145% FPL. The correct figure is roughly $80,000, and the effective 216% older-child limit is about $83,500. MAGI of about $82,300 (2.13x FPL) is under that limit, so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used roughly the right 211% FPL limit but converted it to only about $66,000 for five people; the correct figure is about $81,600, or about $83,500 at the effective 216%. It also tested gross wages of $85,209 instead of MAGI of $82,301 after the traditional 401(k)/IRA deductions. MAGI is 2.13x FPL, which is under the limit."
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used 133% FPL as the Medicaid limit for ages 6-18 and treated the 211% tier as CHIP, even though NC moved that band into Medicaid in 2023. It also tested gross wages of $85,209 instead of MAGI of $82,301 and left out the 5% disregard. MAGI is 2.13x FPL, under the 216% effective limit."
-us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put household income at about 260% FPL; the correct figure is 2.13x FPL, based on $82,301 in MAGI after subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also assumed a lower child limit, but NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the disregard)."
-us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It overstated income at about 219% FPL; MAGI is $82,301, or 2.13x the 2026 FPL for five, once pre-tax traditional 401(k)/IRA contributions are subtracted from wages. It then treated that as above the child limit. NC's older-child Medicaid limit is 216% FPL with the 5% disregard, so 2.13x is eligible."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It treated Child 3's employer-sponsored coverage as disqualifying for Medicaid, but ESI does not block Medicaid eligibility. It also used a stale $36,580 FPL and tested gross wages of $85,209 against 210%. The correct test is MAGI of $82,301 (2.13x the 2026 FPL) against NC's older-child limit of 211% FPL, 216% with the disregard."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 138% FPL limit for ages 6-18, even though NC has covered that group in Medicaid up to 211% FPL (216% with the disregard) since the 2023 NC Health Choice merger. It also double-counted the $7,746 overtime premium, which is already inside the $85,209 wages, and got income above $92,000. Actual MAGI is $82,301."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its income figure of about $82,000 was correct, but it tested that against a 133% FPL child limit. NC's older-child Medicaid limit is 211% FPL (216% effective), and $82,301 is 2.13x FPL, so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a 210% FPL limit with no 5% MAGI disregard and never worked out the household's actual ratio. MAGI of $82,301 is 2.13x FPL, which is under the effective older-child limit of 216% (211% plus 5 points)."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but compared it with 133% FPL ($51,205). NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the disregard), and 2.13x FPL falls under that."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but applied too low a child limit. $82,301 is 2.13x the 2026 FPL for five, which is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It tested gross wages of $85,209 against 210% FPL without subtracting $2,908 in traditional 401(k)/IRA contributions and without the 5% disregard. MAGI of $82,301 is 2.13x FPL, under the effective 216% older-child limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18; NC's older-child Medicaid category runs to 211% FPL (216% with the disregard). It also got MAGI of $61,093 by wrongly subtracting the head's $21,208 ESI premiums, though even the correct $82,301 (2.13x FPL) passes the real limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said income exceeded the limit without giving a figure. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% effective), so a 'no' implies a pre-2023 133%/138% limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but used a 133% FPL Medicaid limit and moved the child into CHIP. Since NC Health Choice merged into Medicaid in 2023, children 6-18 up to 211% FPL (216% effective) are in Medicaid, and 2.13x FPL qualifies."
-us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It compared income with a 133% FPL child limit; NC's older-child Medicaid limit is 211% FPL (216% with the disregard). It also got income of $61,093 by subtracting the head's ESI premiums, though the correct MAGI of $82,301 (2.13x FPL) also qualifies."
-us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard), so a 'no' implies the outdated 133%/138% child threshold."
-us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's limit without giving numbers. MAGI of $82,301 is 2.13x FPL, which is below NC's older-child Medicaid limit of 211% FPL (216% effective) for ages 6-18."
-us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave a generic over-income conclusion. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the disregard), so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's child limits without a figure. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% effective), and household MAGI is 2.13x FPL, so the child qualifies."
-us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It correctly found about 213% FPL, but it capped Medicaid at 133%, labeled the 210% tier as CHIP, and treated 213% as over it. NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, or 216%, so 213% is eligible. Child 3's ESI coverage does not bar Medicaid."
-us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It had the 211% FPL limit right but compared it with gross wages of $85,209 instead of MAGI of $82,301 and left out the 5% disregard. MAGI is 2.13x FPL, under the effective 216% older-child limit."
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave a generic over-threshold conclusion. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the disregard), so the school-age child is eligible."
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It ruled Child 3 ineligible from the wage level without testing against NC's actual child limit. Household MAGI of $82,301 is 2.13x FPL, which is inside NC's older-child Medicaid limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It called earnings of about $85,209 'well above' the NC child limit. After the traditional 401(k)/IRA deductions, MAGI is $82,301, or 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% with the disregard)."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used an estimated threshold that was too low. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% effective), and household MAGI of $82,301 is 2.13x FPL, so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It concluded that MAGI exceeded the limit for a 9-year-old, but MAGI of $82,301 is 2.13x FPL. That is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was above the NC child limit without figures. MAGI is 2.13x FPL, which is below NC's post-2023 older-child Medicaid limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It applied too low an age-specific limit. For ages 6-18, NC's older-child Medicaid category reaches 211% FPL (216% with the disregard), and household MAGI is 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It said income exceeded the child Medicaid limit, but household MAGI of $82,301 is 2.13x FPL. That is under NC's older-child limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but misjudged the limit. $82,301 is 2.13x the 2026 FPL for five, under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)."
-us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It treated Child 3's ESI as a bar to Medicaid, but employer coverage does not block Medicaid eligibility. It also applied too low an income limit: MAGI of $82,301 is 2.13x FPL, under NC's older-child limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It used 133% FPL as NC's Medicaid limit for ages 6-18. Since the 2023 NC Health Choice merger, the older-child category covers up to 211% FPL (216% with the disregard), and MAGI of 2.13x FPL qualifies."
-us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18; NC's older-child Medicaid limit is 211% FPL (216% effective). It also understated MAGI at $74,555; the correct figure is $82,301, which is still eligible at 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It said MAGI exceeded NC's limit for ages 6-18, but that limit is 211% FPL (216% with the 5% disregard) and household MAGI is 2.13x FPL, so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It used a 133% base limit and overstated income at about 240% FPL. Actual MAGI is $82,301, or 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18 and counted employer coverage against eligibility. NC's older-child Medicaid category covers up to 211% FPL (216% with the disregard), MAGI is 2.13x FPL, and ESI does not bar Medicaid."
-us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for child3_medicaid_eligible, so there is nothing to score. The correct answer is eligible: MAGI of 2.13x FPL is under NC's older-child Medicaid limit."
-us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133% (138% with disregard) child limit and overstated MAGI at about 226% FPL. Actual MAGI is 2.13x FPL, and NC's older-child Medicaid limit is 211% FPL (216% with the disregard), so Child 3 is eligible."
-us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income was 'well above' NC's child limits. MAGI of $82,301 is 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% effective)."
-us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed Child 3 in NC's 6-18 group but capped that group at 133% FPL. After the 2023 CHIP merger, the limit is 211% FPL (216% with the disregard), so even its own estimate of about 198% FPL, and the actual 2.13x, qualify."
-us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 133% FPL limit, an FPL for five that was too low ($33,700; the 2026 figure is about $38,680), and gross wages instead of MAGI. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% effective), and the children's ESI does not bar Medicaid."
-us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called income too high without applying NC's actual limit. MAGI of $82,301 is 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL Medicaid limit for ages 6-18 and sent 138-211% FPL to a separate CHIP. Since NC Health Choice merged into Medicaid in 2023, that band is Medicaid. Even its own estimate of about 197% FPL falls inside NC's 211%/216% Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model correctly computed income at about 213% FPL but used the old 133% FPL limit for children 6-18. NC's current Medicaid limit for that age group is 211% FPL plus the 5% disregard (216%), so 213% qualifies."
+us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model turned a 207% FPL limit into $56,232, which is the dollar figure for a much smaller household. For five people, the limit is about $83,000 (216% of the 2026 FPL). It also failed to subtract the $130 IRA deduction, and its MAGI of about $82,300 is below that limit."
+us,scenario_026,child3_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said income was 'well above' the limit without giving a figure. MAGI is $82,301, or 2.13 times FPL, which is below NC's 211%/216% FPL Medicaid limit for older children. Its answer matches using the old 133% limit."
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model named the right 211% FPL limit but put it at about $66,000 for a family of five, when 211-216% of the 2026 FPL is about $81,500-$83,400. It also compared gross wages of $85,209 instead of MAGI of $82,301, which is net of the 401(k) and IRA deductions."
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated 133% FPL as the Medicaid limit and 211% as a separate CHIP limit, but NC Health Choice was merged into Medicaid. It then compared gross wages of $85,209, not MAGI of $82,301, to an unadjusted 211% figure, leaving out the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model put income at about 260% FPL, but MAGI of $82,301 is 2.13 times the 2026 FPL for five people. That is within NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model's 219% FPL figure comes from dividing $82,301 by the 2025 FPL for five ($37,650) instead of the 2026 guideline, which gives 2.13. It also left out the 5% disregard on NC's 211% Medicaid limit for older children, so it treated an eligible income as too high."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model ruled Child 3 out because of employer-sponsored insurance, but Medicaid has no ESI exclusion; the uninsured requirement applies only to CHIP. It also tested gross wages of $85,209 against a 210% limit built on a low FPL ($36,580), instead of testing MAGI of $82,301 (2.13 times FPL) against 216%."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 138% FPL limit for ages 6-18, but NC Medicaid now covers older children up to 211%/216% FPL. It also counted the $7,746 overtime twice, reaching more than $92,000, even though the $85,209 gross wages already include overtime."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model's MAGI of about $82,000 is right, but it tested it against the old 133% FPL limit. NC's Medicaid limit for older children is 211% plus the 5% disregard, and $82,301 is 2.13 times FPL, so Child 3 qualifies."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a 210% FPL limit with no 5% MAGI disregard and implicitly used gross income. MAGI of $82,301 is 2.13 times FPL, which is under NC's effective 216% Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it compared it to the outdated 133% FPL limit ($51,205). Since the NC Health Choice merger, NC Medicaid covers children 6-18 up to 211% FPL plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly found MAGI of $82,301 but said it exceeds the limit. That amount is 2.13 times the 2026 FPL for five, below NC's 211%/216% FPL Medicaid limit for older children, so the model used a lower, pre-merger limit."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model compared gross wages of $85,209 to a 210% limit. It should have used MAGI of $82,301, after the $2,778 401(k) and $130 IRA deductions, and added the 5% disregard. At 2.13 times FPL, the child is eligible."
+us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model subtracted Head's $21,208 ESI premiums from wages to reach a MAGI of $61,093, but MAGI is $82,301. It then used the outdated 133% FPL limit instead of NC's 211%/216% Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income was above the limit without naming one. MAGI of $82,301 (2.13 times FPL) is within NC's 216% FPL Medicaid limit for children 6-18, so the answer matches using the old 133% limit."
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it used a 133% FPL Medicaid limit and sent the child to CHIP. NC Health Choice was absorbed into Medicaid in 2023, so children 6-18 are Medicaid-eligible up to 211% FPL plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly subtracted the $21,208 ESI premiums to get a $61,093 income figure, and then applied the outdated 133% FPL limit. The real MAGI of $82,301 (2.13 times FPL) is under NC's 216% Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. MAGI of $82,301 is 2.13 times FPL, within NC's 211%/216% FPL Medicaid limit for older children. A 'not eligible' answer matches using the old 133% limit."
+us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's limit without naming it. MAGI of $82,301 is 2.13 times FPL, below the 216% FPL Medicaid limit for children 6-18 that has applied since the NC Health Choice merger."
+us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model gave no threshold. MAGI of $82,301 (2.13 times FPL) is within NC's 211%-plus-5%-disregard Medicaid limit for older children, so the answer matches using the old 133% limit."
+us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's limits without a figure. MAGI is 2.13 times FPL, below NC's 216% FPL Medicaid limit for children 6-18, so the model used an outdated, lower limit."
+us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model correctly put income at about 213% FPL but split coverage into 133% Medicaid and 210% CHIP. Since the 2023 merger, NC Medicaid itself covers older children up to 211% plus the 5% disregard (216%). It also wrongly treated ESI as a bar to Medicaid."
+us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used the right 211% limit but compared gross wages of $85,209 instead of MAGI of $82,301, which is net of the 401(k) and IRA deductions. It also left out the 5% disregard. At 2.13 times FPL, the child is eligible."
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said earnings were above the threshold without naming one. MAGI of $82,301 (2.13 times FPL) is within NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model judged income too high without doing any calculation. MAGI is 2.13 times the 2026 FPL for five, below NC's 211%/216% FPL Medicaid limit for children 6-18."
+us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model judged gross earnings of about $85,209 to be 'well above' the limit. MAGI after the 401(k) and IRA deductions is $82,301, or 2.13 times FPL, which is within NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's estimated NC threshold is too low. NC Medicaid covers children 6-18 up to 211% FPL plus the 5% disregard, and this household's MAGI is 2.13 times FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeds NC's limit for a 9-year-old. MAGI of $82,301 is 2.13 times FPL, below the 216% FPL Medicaid limit for older children that has applied since NC Health Choice merged into Medicaid."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model said income exceeds the limit without naming it. MAGI is 2.13 times FPL, within NC's 211%/216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied too low a Medicaid limit for age 9. NC covers children 6-18 up to 211% FPL plus the 5% disregard, and MAGI of $82,301 is 2.13 times FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeds the child Medicaid limit. MAGI of 2.13 times FPL is below NC's 216% FPL Medicaid limit for older children, so the model used the old, lower limit."
+us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it said this exceeds NC's limit. That amount is 2.13 times the 2026 FPL for five, under NC's 211%-plus-5%-disregard Medicaid limit for children 6-18."
+us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model cited employer coverage as disqualifying, but Medicaid has no ESI exclusion. It also understated the income limit: MAGI of 2.13 times FPL is within NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit for ages 6-18. Since NC Health Choice merged into Medicaid, the limit is 211% plus the 5% disregard, which MAGI of 2.13 times FPL meets."
+us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model used an incorrect MAGI of $74,555 (the actual figure is $82,301) and tested it against the outdated 133% FPL limit. NC's current Medicaid limit for children 6-18 is 211%/216% FPL, and the household is at 2.13 times FPL."
+us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model said MAGI exceeds the limit for ages 6-18. MAGI is 2.13 times FPL, below NC's 216% FPL Medicaid limit for older children, so the model used the old 133% limit."
+us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model overstated income at about 240% FPL (actual 2.13) and used the old 133% limit. NC's Medicaid limit for older children is now 211% FPL plus the 5% disregard.
+us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit for ages 6-18 and treated ESI as disqualifying, which it is not for Medicaid. MAGI of 2.13 times FPL is within NC's 211%/216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for child3_medicaid_eligible, so its answer could not be scored."
+us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133%/138% FPL Medicaid limit for children 6-18 and overstated income at 226% FPL. MAGI is 2.13 times the 2026 FPL, under NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model said income is well above NC's children's Medicaid limit. MAGI of $82,301 is 2.13 times FPL, within the 211%-plus-5%-disregard limit for ages 6-18."
+us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed ages 6-18 in a 133% FPL Medicaid group, but NC's limit has been 211% plus the 5% disregard since the Health Choice merger. Its own estimate of about 198% FPL is under that limit."
+us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit with a too-low FPL of $33,700 for five, and compared gross wages instead of MAGI. MAGI of $82,301 is 2.13 times the 2026 FPL, within NC's 216% FPL Medicaid limit for older children."
+us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model said income is too high without naming a limit. MAGI is 2.13 times FPL, below NC's 211%/216% FPL Medicaid limit for children 6-18."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It capped the nonrefundable child tax credit at $1,500 for three children; the 2026 CTC is $2,200 per child ($6,600 total) and its nonrefundable portion is applied against the full pre-credit liability before any refundable ACTC is computed, so the $4,586.51 of tax is entirely absorbed. It also used a $29,500 MFJ standard deduction instead of $32,200 and never took the $7,746.26 deduction for the qualified FLSA overtime premium. Its own stated arithmetic ($6,313 less $1,500 = $4,813) does not equal the $5,813 it submitted."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its derivation reached the correct conclusion — the nonrefundable CTC absorbs the entire tentative tax, 'Result near $0' — and it then submitted $4,107, a figure no step in its own work produces. It also omitted the $7,746.26 qualified-overtime deduction that brings taxable income down to $42,354.26 and tentative tax to $4,586.51."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It invented a ~$605 ceiling on the nonrefundable CTC by treating the $1,700-per-child refundable ACTC as consumed first; the nonrefundable credit is applied against the whole liability first and the refundable portion is the residual, so $6,600 of credit for three children eliminates the entire tax. It also missed the $7,746.26 qualified-overtime deduction, which lowers taxable income from its stated $50,001 to $42,354.26."
@@ -2148,97 +2222,100 @@ us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,
us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of the $85,209 gross wages that already include it, then applied pre-TCJA parameters — a $16,200 standard deduction, $26,000 of personal exemptions, a 15% bracket, and a $1,000-per-child CTC. For 2026 the premium is instead deducted, the standard deduction is $32,200 with no exemptions, and the $6,600 credit for three children offsets the full $4,586.51 tentative tax."
us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation reached the grader. The correct derivation yields $42,354.26 of taxable income after the $32,200 standard deduction and $7,746.26 overtime deduction, $4,586.51 of tentative tax, and $0 after the $6,600 nonrefundable child tax credit."
us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a fabricated $3,700 MFJ standard deduction in place of $32,200 and a fabricated $3,700 total child tax credit in place of $6,600, and it never subtracted the $2,778 traditional 401(k), the $130 IRA deduction, or the $7,746.26 overtime deduction. Its explanation states the credit fully offsets the tax while it submitted $5,931.15, contradicting itself."
-us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It estimated pre-credit tax at about $5,500, which is less than the $6,600 CTC, but still said the full credit was used nonrefundably and paid $0 ACTC. Even by its own numbers, about $1,100 was unused and refundable. It also left out the $7,746 qualified overtime deduction, which lowers tax to $4,586.51 and leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It made up a refundable amount of about $1,000 per child and never computed tax liability. ACTC is the CTC left over after it offsets tax: $6,600 minus $4,586.51 of pre-credit tax, which is $2,013.49, within the $1,700-per-child cap."
-us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) and the 2025 $31,500 standard deduction. It then called a $6,000 credit 'fully absorbed' by about $5,600 of tax, even though $400 was left over. With the $2,200-per-child CTC, the $32,200 deduction and the $7,746 overtime deduction, tax is $4,586.51 and $2,013.49 is refundable."
-us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said tax was high enough to absorb a $6,000 CTC without computing it, and it used $2,000 per child instead of the 2026 $2,200. After the $32,200 standard deduction and the $7,746 qualified overtime deduction, pre-credit tax is only $4,586.51, so $2,013.49 of the $6,600 CTC is refundable."
-us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It correctly used the $6,600 CTC and the credit ordering, but it put pre-credit tax at about $5,450 because it left out the OBBBA deduction for the $7,746 FLSA overtime premium. With that deduction, taxable income is about $42,355, tax is $4,586.51, and refundable ACTC is $2,013.49 rather than $1,150."
-us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used pre-OBBBA and 2025 values: a $2,000-per-child CTC ($6,000), a roughly $30,000 standard deduction and a $23,850 10% bracket. It also left out the $7,746 qualified overtime deduction, which gave tax of $5,815 and ACTC of $185. The 2026 values ($2,200 per child, $32,200 deduction, $24,800 bracket) plus the overtime deduction give tax of $4,586.51 and ACTC of $2,013.49."
-us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime on top of $85,209 in wages that already include it, so it put AGI at about $89,000. It should instead have taken the overtime premium as the OBBBA qualified overtime deduction. It also used a $6,000 CTC and asserted that tax exceeded it, when the correct pre-credit tax of $4,586.51 leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is the correct computation with the $378 of auto-loan interest also deducted: taxable income of $41,977 instead of $42,355. The OBBBA passenger-vehicle loan interest deduction requires a new, US-assembled vehicle, and no such fact is listed, so the deduction is unavailable. Tax is $4,586.51 and ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC and assumed it was fully used against tax. The 2026 CTC is $6,600, and pre-credit tax after the $32,200 standard deduction and $7,746 overtime deduction is only $4,586.51, which leaves $2,013.49 of refundable ACTC."
-us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It asserted that the nonrefundable CTC fully offsets tax with nothing left over, which reverses the actual relationship. With AGI of $82,301, the $32,200 standard deduction and the $7,746 qualified overtime deduction, tax is $4,586.51, below the $6,600 CTC, so $2,013.49 is refundable ACTC."
-us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, the TCJA-sunset amount), but OBBBA sets the 2026 CTC at $2,200 per child ($6,600). That total exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable."
-us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its $5,516.12 tax is the correct 2026 computation without the $7,746 qualified overtime deduction, and it used a $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600). The overtime deduction lowers tax to $4,586.51, and the $6,600 credit leaves $2,013.49 refundable instead of $483.88."
-us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used a $1,000-per-child CTC with the pre-TCJA refundability formula. OBBBA made the CTC $2,200 per child, refundable up to $1,700, and kept the TCJA rates and standard deduction, so the unused $2,013.49 above the $4,586.51 tax is refundable."
-us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no computation and simply said no refundable credits apply. The $6,600 CTC exceeds the $4,586.51 pre-credit tax (after the $32,200 standard deduction and $7,746 overtime deduction), so $2,013.49 is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed AGI correctly at $82,301 but used a $3,000 CTC (the $1,000-per-child sunset amount) instead of the OBBBA $2,200 per child. The $6,600 CTC exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $61,093 AGI is the true $82,301 minus the Head's $21,208 of employer-sponsored insurance premiums, which it wrongly subtracted. That produced a spurious $930 EITC. With the actual $82,301 AGI, EITC is zero, and ACTC is the $2,013.49 of CTC left after $4,586.51 of tax."
-us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation for its $0 answer. The correct pre-credit tax is $4,586.51 (AGI of $82,301, minus the $32,200 standard deduction and $7,746 overtime deduction), which leaves $2,013.49 of the $6,600 CTC as refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its explanation only restates the variable and gives no computation. Its $0 answer misses that the $6,600 CTC exceeds the $4,586.51 pre-credit tax, which leaves $2,013.49 of refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It correctly noted that the nonrefundable CTC covers the tax, but it did not refund the unused remainder as ACTC. Of the $6,600 credit, $4,586.51 offsets tax and the remaining $2,013.49 is refundable within the $1,700-per-child cap."
-us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,other,False,"It said the nonrefundable CTC was fully absorbed by pre-credit tax, but the $4,586.51 tax is smaller than the $6,600 CTC. The unused $2,013.49 is refundable as ACTC."
-us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC ($2,000 per child) and a $5,676.12 pre-credit tax that leaves out the $7,746 qualified overtime deduction. With the $6,600 CTC and tax of $4,586.51, ACTC is $2,013.49 instead of $323.88."
-us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,other,False,"It treated the $1,700-per-child refundable cap ($5,100) as the ACTC amount. The cap only limits the refundable portion. ACTC is the CTC left after offsetting tax, which is $6,600 minus $4,586.51, or $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its reasoning about the children being over age 6 is irrelevant to the CTC, and it never compared the credit with tax liability. The $6,600 CTC exceeds the $4,586.51 pre-credit tax, which leaves $2,013.49 of refundable ACTC."
-us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed without computation that no refundable credits apply at this income. The three children qualify for a $6,600 CTC, and the pre-credit tax is only $4,586.51, so $2,013.49 is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the TCJA-sunset rules: a $1,000-per-child CTC treated as nonrefundable. OBBBA sets the 2026 CTC at $2,200 per child with up to $1,700 refundable, so the $2,013.49 of credit above the $4,586.51 tax is refunded."
-us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $2,058.76 means pre-credit tax of $4,541.24, which is the correct computation with the $378 of auto-loan interest also deducted. The OBBBA vehicle-loan interest deduction requires a new, US-assembled vehicle, and no such fact is listed, so tax is $4,586.51 and ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It put regular tax at about $5,532, which leaves out the OBBBA deduction for the $7,746 FLSA overtime premium. With that deduction, tax is $4,586.51 and the unused CTC is $2,013.49, not $1,068."
-us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 tax is the correct computation with the $378 of auto-loan interest also deducted (taxable income $41,977 instead of $42,355). The OBBBA vehicle-loan interest deduction requires a new, US-assembled vehicle that the facts do not establish. Tax is $4,586.51 and ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,other,False,"It declared no refundable CTC at this income without computing anything. The CTC has no phase-out near $82,301 AGI, and the $6,600 credit exceeds the $4,586.51 pre-credit tax, so $2,013.49 is refundable."
-us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 CTC (the $1,000-per-child sunset amount) instead of the OBBBA $2,200 per child. A $6,600 CTC exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable ACTC."
-us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC and ordering, but overstated pre-credit tax at $4,671. Wages minus the $2,778 401(k), the $130 IRA, the $32,200 standard deduction and the $7,746 overtime deduction give taxable income of about $42,355 and tax of $4,586.51, so ACTC is $2,013.49 rather than $1,929."
-us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a $200 Saver's Credit before the CTC, but AGI of $82,301 is above the 2026 MFJ Saver's Credit limit of $80,500, so that credit is zero. All $4,586.51 of pre-credit tax is offset by the CTC, leaving $2,013.49 refundable, not $2,205."
-us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,000 CTC instead of the OBBBA $6,600, and it added overtime on top of wages that already include it, which put tax at $5,937. It should have deducted the $7,746 overtime premium, giving tax of $4,586.51 and leaving $2,013.49 of refundable ACTC."
-us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits. The required output of $2,013.49 in refundable ACTC was never submitted."
-us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly used the $6,600 CTC and the earned-income test, but put pre-credit tax at $5,873, leaving out the OBBBA deduction for the $7,746 overtime premium. With that deduction and the $32,200 standard deduction, tax is $4,586.51 and ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,other,False,"It said additional CTC is $0 without computing tax liability. The $6,600 CTC exceeds the $4,586.51 pre-credit tax, so $2,013.49 is refundable ACTC."
-us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,other,False,"Its $5,586.60 pre-credit tax is exactly $1,000 more than the $4,586.60 that taxable income of $42,355 produces at the 10%/12% rates, which is an arithmetic slip in the bracket computation. The correct tax leaves about $2,013.49 of refundable ACTC rather than $1,013.40."
-us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) and a $5,479 tax that leaves out the $7,746 qualified overtime deduction. The 2026 $6,600 CTC less $4,586.51 of tax leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It gave an unexplained $4,300 ACTC based only on the number of children and never subtracted the part of the CTC used against tax. ACTC is the $6,600 CTC minus the $4,586.51 of pre-credit tax, or $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It estimated tax at about $5,500 and then said the full $6,600 CTC was used nonrefundably, even though $5,500 is less than $6,600. It also skipped the $7,746 qualified overtime deduction, which lowers tax to about $4,587. That leaves $2,013.49 of CTC as refundable ACTC."
+us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It assigned an arbitrary $1,000 refundable amount per child and never figured tax liability. The refundable CTC is the $6,600 credit minus the roughly $4,586.51 used against tax (after the standard and overtime deductions), which is $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It said the nonrefundable CTC wipes out any refundable portion. However, pre-credit tax is only about $4,587 after the $32,200 standard deduction and the $7,746 overtime deduction, which is less than the $6,600 CTC. The $2,013.49 difference is refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000 per child instead of $2,200 and a $31,500 standard deduction. It also left out the $7,746 qualified overtime deduction. Its own $5,600 tax is still below its $6,000 credit, yet it called the credit fully absorbed. The correct figures are about $4,586.51 in tax against a $6,600 CTC, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used $2,000 per child instead of the OBBBA $2,200 and assumed tax was high enough to absorb the credit. It never subtracted the $7,746 overtime deduction. Pre-credit tax is about $4,587, so $2,013.49 of the $6,600 CTC is refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but left out the $7,746 FLSA overtime premium deduction, which overstated tax at about $5,450. With that deduction, tax is about $4,586.51, so the refundable ACTC is $2,013.49, not $1,150."
+us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC, a guessed $30,000 standard deduction instead of $32,200, and 2025 brackets. It also skipped the $7,746 qualified overtime deduction, which gave it $5,815 in tax. The correct tax is about $4,586.51, leaving $2,013.49 of the $6,600 CTC refundable."
+us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of gross wages, even though gross wages already include it. The overtime premium should instead be a below-the-line deduction. It also used $2,000 per child and wrongly subtracted Roth contributions. Correct taxable income is about $42,355, giving $4,586.51 in tax and $2,013.49 of refundable CTC from the $6,600 credit."
+us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is $45.36 lower than the correct $4,586.51, which equals 12% of $378. That means it took the OBBBA car-loan interest deduction for the $378 auto loan interest. No qualifying new U.S.-assembled vehicle is listed, so that deduction doesn't apply and the refundable CTC is $2,013.49."
+us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of the 2026 OBBBA $6,600 and said tax used all of it. Because it ignored the $7,746 overtime deduction, it missed that tax is only about $4,587. That leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It got AGI of $82,301 right but said the nonrefundable CTC fully offsets tax with nothing left over. After the $32,200 standard deduction and the $7,746 overtime deduction, tax is about $4,586.51. The $6,600 CTC exceeds that by $2,013.49, which is refundable."
+us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child credit ($3,000) as if the TCJA had expired. OBBBA made the CTC permanent at $2,200 per child, or $6,600, which exceeds the roughly $4,587 tax by $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,516.12 tax correctly reflects the $32,200 standard deduction but leaves out the $7,746 qualified overtime deduction. It also used $2,000 per child instead of $2,200. The correct figures are $4,586.51 in tax against a $6,600 CTC, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It reverted to 2017 rules, with a $1,000-per-child CTC and the $3,000 refundability threshold, instead of applying OBBBA's permanent $2,200-per-child credit and the overtime deduction. The correct figures are $6,600 in CTC against about $4,586.51 in tax, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It gave no reasoning, and its $0 answer treats the CTC as fully absorbed by tax. Tax is about $4,586.51 after the $32,200 standard deduction and the $7,746 overtime deduction. The $6,600 CTC exceeds that, leaving $2,013.49 of refundable ACTC."
+us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child under the expired TCJA sunset) instead of OBBBA's $2,200 per child. The $6,600 credit exceeds the roughly $4,586.51 tax, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from wages, which gave it an AGI of $61,093 and a spurious $930 EITC. AGI is actually $82,301, so EITC is $0. The only refundable credit is the $2,013.49 of CTC left after about $4,586.51 of tax."
+us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation, and its $0 answer ignores the refundable CTC. The $6,600 CTC ($2,200 for each of three children) exceeds the roughly $4,586.51 tax (after the standard deduction and the $7,746 overtime deduction) by $2,013.49, which is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It gave no derivation, and its $0 answer treats the CTC as entirely nonrefundable or used up. Tax is only about $4,586.51, so $2,013.49 of the $6,600 CTC is refundable as ACTC."
+us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It claimed the nonrefundable CTC covers tax with nothing left over. Pre-credit tax is about $4,586.51 after the $32,200 standard deduction and the $7,746 qualified overtime deduction, so the $6,600 CTC leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It said tax fully absorbs the CTC, but it never compared the two amounts. Tax after the standard and overtime deductions is about $4,586.51, below the $6,600 CTC, which leaves $2,013.49 of refundable ACTC."
+us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of $6,600 and calculated $5,676.12 in tax without the $7,746 qualified overtime deduction. The correct tax of $4,586.51 leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It paid the full $1,700-per-child refundable cap ($5,100) without first applying the CTC against tax. The ACTC is limited to the credit left after tax, which is $6,600 − $4,586.51 = $2,013.49."
+us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It ruled out EITC correctly but gave no refundable CTC at all. The $6,600 CTC exceeds the roughly $4,586.51 pre-credit tax by $2,013.49, and that amount is refundable within the $1,700-per-child cap."
+us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed no refundable credits apply and never figured the ACTC. With three qualifying children and tax of about $4,586.51, $2,013.49 of the $6,600 CTC remains and is refundable."
+us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied post-2025 sunset rules with a nonrefundable $1,000 credit per child. OBBBA permanently set the CTC at $2,200 per child with up to $1,700 refundable, which produces $2,013.49 of refundable ACTC here."
+us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its result implies pre-credit tax of $4,541.24, which is $45.36 (12% of $378) below the correct $4,586.51. It deducted the $378 auto loan interest as OBBBA qualified vehicle-loan interest, which doesn't apply here. The refundable CTC is $2,013.49."
+us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but estimated tax at about $5,532 without the $7,746 qualified overtime deduction. With that deduction, tax is about $4,586.51 and the refundable ACTC is $2,013.49."
+us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is $45.36 too low because it deducted the $378 auto loan interest as qualified passenger-vehicle loan interest. The correct tax is $4,586.51, which leaves $2,013.49 of the $6,600 CTC refundable."
+us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It said this income level doesn't qualify for refundable CTC. ACTC doesn't phase out at $82,301 AGI. The $6,600 CTC exceeds the roughly $4,586.51 tax, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child under the TCJA sunset) instead of OBBBA's $2,200 per child. The $6,600 credit exceeds the roughly $4,586.51 tax by $2,013.49, which is refundable."
+us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its $4,671 pre-credit tax matches using the 2025 $31,500 MFJ standard deduction instead of the 2026 $32,200, which overstates tax by about $84. The correct tax is $4,586.51, leaving $2,013.49 refundable rather than $1,929."
+us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a $200 Saver's Credit before the CTC. AGI of $82,301 exceeds the 2026 MFJ Saver's Credit income limit ($80,500), so no Saver's Credit applies. The CTC offsets the full $4,586.51 tax, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child) instead of OBBBA's $6,600. It also added the overtime premium to wages ($90,047) instead of deducting it, which pushed tax to $5,937. The correct tax is about $4,586.51, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_refundable_credits, so its answer is missing."
+us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but calculated $5,873 in tax without the $7,746 qualified overtime deduction. With that deduction, tax is $4,586.51 and the refundable ACTC is $2,013.49."
+us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It said the additional CTC is $0 without comparing the credit with tax. The $6,600 CTC exceeds the roughly $4,586.51 pre-credit tax by $2,013.49, which is refundable."
+us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its $5,586.60 pre-credit tax is about $1,000 too high because it left out the $7,746 FLSA overtime premium deduction. The correct tax is $4,586.51, so $2,013.49 of the $6,600 CTC is refundable."
+us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used $2,000 per child ($6,000) instead of OBBBA's $2,200 per child. It also calculated $5,479 in tax without the $7,746 overtime deduction. The correct figures are $6,600 in credit and $4,586.51 in tax, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It gave an unsupported $4,300 ACTC without capping refundability at the credit left after offsetting tax. The refundable CTC is $6,600 − $4,586.51 = $2,013.49."
us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price."
us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed federal employee FICA correctly ($5,283.00 Social Security + $1,235.53 Medicare on $85,209) and then invented a ~$1,016.75 ""NC state payroll tax"" by mixing North Carolina's flat individual income tax rate into the payroll-tax output. North Carolina imposes no mandatory employee-side payroll tax — its unemployment insurance tax is employer-paid — so the state add-on is entirely spurious and accounts for the whole $1,016.81 overage."
-us,scenario_026,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It set the FICA base at $92,955 by adding the spouse's $7,746 FSLA overtime premium to the $85,209 of gross wages, double-counting overtime that the prompt states is already included in the annual gross wage total. The overtime premium is a decomposition of the same wages, not additional earnings, so the base is $85,209 and 7.65% of it is $6,518.49, not the $7,110 it reported."
-us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It made two compounding base errors: it added the $7,746 FSLA overtime premium to the $85,209 gross wages (overtime is already inside that annual total), then subtracted the Head's $21,208 employer-sponsored insurance premiums as a Section 125 pre-tax exclusion. The ESI premium belongs to the Head, who has no wages at all, and PolicyEngine computes employee Social Security and Medicare tax on gross employment income with no health-premium exclusion, so the base stays $85,209 and the tax is $6,518.49."
-us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It explicitly built the base from ""$85,209 wages and $7,746 overtime,"" double-counting the FSLA overtime premium that is already contained in the annual gross wage figure. Its $6,732.19 does not even follow from that inflated base (7.65% of $92,955 is $7,111.06), so it submitted an unreconciled estimate instead of the $6,518.49 that 6.2% plus 1.45% on $85,209 yields."
-us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the Head's $21,208 employer-sponsored insurance premiums from the Spouse's $85,209 wages as a cafeteria-plan pre-tax exclusion, taxing only $64,001 at 7.65%. The premiums are recorded on the Head, who has zero wages, and PolicyEngine's employee Social Security and Medicare tax applies to gross employment income without any ESI exclusion, so the full $85,209 is taxable and the answer is $6,518.49."
-us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It applied every rule correctly — 6.2% Social Security and 1.45% Medicare on the Spouse's $85,209, no Additional Medicare Tax below the $250,000 MFJ threshold, and no mandatory North Carolina employee payroll tax — but multiplied out wrong. The exact product is $5,282.96 + $1,235.53 = $6,518.49; its $6,519.97 overstates the Medicare component by about $1.48, an arithmetic slip rather than a rule error."
-us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It gave no derivation, only a rate recital, and its $10,618 is 63% above the employee-side liability; the correct computation on $85,209 is 6.2% ($5,282.96) plus 1.45% ($1,235.53) = $6,518.49. The value tracks the combined employer-plus-employee Social Security rate of 12.4% on $85,209 ($10,565.92), meaning it folded in the employer share that the output definition explicitly excludes."
-us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It stated the base as ""total wages $92,955,"" adding the $7,746 FSLA overtime premium on top of the $85,209 gross wages even though the prompt specifies that gross wage totals already include overtime pay. Its rate handling and its correct finding that North Carolina levies no employee payroll tax were fine; the single $7,746 double-count inflates the answer by $592.57 over the correct $6,518.49."
-us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key never reached the outputs object. This is a missing-submission failure rather than a substantive modeling error."
-us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It zeroed out the entire wage base by asserting that ""the employer portion covering the head"" and modest household income leave no employee liability. Employee Social Security and Medicare tax are flat, unconditional levies on wages with no income floor and no offset for employer contributions or employer-paid health premiums, so the Spouse's $85,209 generates $5,282.96 + $1,235.53 = $6,518.49."
+us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got FICA on $85,209 right ($5,283.00 + $1,235.53) but then added $1,016.75 of North Carolina income tax and called it a state payroll tax. NC has no mandatory employee state payroll tax, so the correct total is FICA alone: $6,518.47."
+us,scenario_026,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model added the $7,746 overtime premium on top of the $85,209 gross wages, even though gross wages already include overtime. That gave a $92,955 base and about $7,112 of FICA. It then submitted $13,330, which matches no step in its own reasoning. The correct answer is 7.65% of $85,209, or $6,518.47."
+us,scenario_026,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model added the $7,746 FLSA overtime premium to the $85,209 gross wages and applied 7.65% FICA to $92,955. The prompt says gross wages already include overtime, so this double-counted overtime. The correct base is $85,209, giving $6,518.47."
+us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model double-counted overtime with a $92,955 base. It then subtracted the $21,208 employer-sponsored insurance premiums as a pre-tax cafeteria-plan exclusion, but those premiums belong to the head, who has no wages, and they don't reduce the FICA base here. Applying 7.65% to the spouse's full $85,209 gives $6,518.47."
+us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model said it applied 6.2% and 1.45% to $85,209 plus $7,746 of overtime, which double-counts overtime already included in gross wages. Its $6,732.19 also doesn't match its own stated base ($92,955 × 7.65% = $7,111.06). The correct figure is 7.65% of $85,209, or $6,518.47."
+us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's $85,209 wages and applied FICA to $64,001. Those premiums don't reduce the earner's Social Security or Medicare wages, so the full $85,209 is the base, giving $6,518.47."
+us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly applied 6.2% Social Security and 1.45% Medicare to the spouse's $85,209 and correctly found no Additional Medicare Tax and no NC payroll tax. It then added the components wrong: $5,282.96 + $1,235.53 = $6,518.49, not $6,519.97, leaving it $1.50 off."
+us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $10,618 implies a FICA base of about $138,800, well above anything the household earns. The only earner has $85,209 in wages, overtime included, so 7.65% gives $6,518.47."
+us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model treated total wages as $92,955 by adding the $7,746 overtime premium to the $85,209 gross wages. The prompt says gross wages already include overtime, so the FICA base is $85,209 and the tax is $6,518.47."
+us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for payroll_tax, so there was no answer to score. The correct derivation is 7.65% employee FICA on the spouse's $85,209 in wages, or $6,518.47."
+us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model said employer coverage and modest household income produce roughly zero employee payroll tax. Employee FICA has no income exemption and isn't offset by employer contributions, so the spouse owes 6.2% plus 1.45% on all $85,209 of wages, or $6,518.47."
us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium from $85,209 of wages to produce $64,001, even though the school-meal income calculation in this case uses $85,209. That income is 220% of the federal poverty guideline, above the 185% reduced-price limit, and no categorical pathway applies."
us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used $500 per child ($1,500 total) for the NC child deduction. At AGI $80k–$100k, joint filers get $1,500 per child, or $4,500. It then submitted $2,882, which does not match its own calculation of $2,206.51."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from federal taxable income using a $29,200 federal standard deduction and applied a 4.75% rate. It then invented a dependent credit and submitted $5,238, which its own work does not support. NC starts from federal AGI, subtracts a $25,500 standard deduction and a $4,500 child deduction, and taxes the rest at 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model left the $130 traditional IRA deduction out of AGI and used $2,500 per child for the child deduction instead of $1,500 per child at the $80k–$100k tier. It also switched between 4.25% and 3.99%, and its final $3,041 matches none of its own calculations."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model left out the NC child deduction ($1,500 × 3 = $4,500) and did not subtract the $130 traditional IRA from AGI. As a result, it taxed $56,931 instead of $52,300.52 at 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an outdated $21,500 joint standard deduction instead of $25,500. It also applied the $3,000-per-child amount from the lowest AGI tier, but at $82,301 AGI the deduction is $1,500 per child. The two errors nearly cancel, which is why its $2,066.86 lands close to the right answer by accident."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the $7,746 FLSA overtime premium on top of gross wages, even though wages already include it. It also left out the $4,500 NC child deduction and used a 4.25% rate instead of 3.99%. Finally, it applied an invented retirement add-back to reach $3,350."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used $1,000 per child, which belongs to the $100k–$120k tier. At $82,301 AGI, joint filers fall in the $80k–$100k tier, which gives $1,500 per child ($4,500 total)."
-us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model added the $7,746 overtime premium to wages, which already include it, giving an AGI of $90,047 instead of $82,300.52. It also left out the $4,500 NC child deduction."
-us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the NC child deduction of $1,500 per child ($4,500). It therefore taxed $56,801 instead of $52,300.52 at 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model left out the $4,500 NC child deduction for three children at the $80k–$100k AGI tier. It also applied a 4.0% rate instead of the 2026 rate of 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the $4,500 NC child deduction. It therefore taxed $56,801 instead of $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the head's $21,208 employer-sponsored insurance premiums from AGI, even though the spouse's gross wages are not reduced by that input, and got an AGI of $61,093 instead of $82,300.52. Because AGI was too low, it then chose the $2,000-per-child tier instead of $1,500."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct method is $82,300.52 AGI minus $30,000 in NC deductions, taxed at 3.99%, which gives $2,086.79. The model's $3,538 corresponds to about $88,700 taxed at 3.99%, which means it effectively applied neither the $25,500 standard deduction nor the $4,500 child deduction."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 employer-sponsored insurance premiums from AGI and got $61,093 instead of $82,300.52. It then applied the $2,000-per-child tier instead of the $1,500-per-child tier that applies at the correct AGI."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct method is $82,300.52 AGI minus the $25,500 standard deduction and $4,500 child deduction, taxed at 3.99%, which gives $2,086.79. The model's $3,538.77 corresponds to about $88,700 taxed at 3.99%, which means it effectively applied none of the NC deductions."
-us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a 3.74% rate instead of the 2026 rate of 3.99% and left out the $4,500 NC child deduction. It also subtracted an extra $130 spousal IRA deduction for the head, who made no IRA contribution."
-us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used an invented $4,000-per-child deduction ($12,000 total). At $82,301 joint AGI, the NC child deduction is $1,500 per child ($4,500)."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed deductions wipe out NC tax. The $25,500 standard deduction plus the $4,500 child deduction total only $30,000 against $82,300.52 of AGI, which leaves $52,300.52 taxable at 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $4,875 is more than double the correct $2,086.79 ($52,300.52 × 3.99%), which means it taxed close to gross wages at a rate well above 3.99% and never applied the $30,000 of NC deductions."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,957.29 equals $49,055 × 3.99%. That means it subtracted the $7,746 overtime premium from AGI, even though overtime remains in federal and NC AGI, and it left out the $4,500 NC child deduction."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $2,027 equals ($82,301 − $25,500 − $6,000) × 3.99%. That means it used the $2,000-per-child tier ($60k–$80k) instead of the $1,500-per-child tier that applies at $80k–$100k joint AGI."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model left out the $4,500 NC child deduction and did not subtract the $130 traditional IRA from AGI. It therefore taxed $56,931 instead of $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used $1,000 per child ($3,000), which is the $100k–$120k tier. At $82,301 joint AGI, the correct amount is $1,500 per child ($4,500)."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model started from about $85k of gross wages, without subtracting the 401(k) and IRA contributions, and left out the $4,500 NC child deduction. Its $2,780 also implies a rate above the 2026 rate of 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the NC child deduction of $1,500 per child ($4,500)."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium to get an AGI of $74,555, but overtime stays in NC AGI. It also left out the $4,500 NC child deduction."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model removed the $7,746 overtime premium from AGI, getting $74,555 instead of $82,300.52. Because of that lower AGI, it applied the $2,000-per-child tier instead of the correct $1,500 per child."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added the overtime premium on top of wages, which already include it, and got an AGI of $90,047. It also left out the $4,500 child deduction and applied 4.25% instead of 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium from AGI and got $74,555. It then applied $2,500 per child, which belongs to the $40k–$60k tier. At the correct AGI of $82,300.52, the deduction is $1,500 per child."
-us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used gross wages of $85,209 as AGI without subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also left out the $4,500 NC child deduction."
-us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied personal exemptions for a family of five, but North Carolina has no personal exemptions. The only deductions are the $25,500 standard deduction and the $4,500 child deduction, which leave $52,300.52 taxable at 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium from AGI and got $74,555, but overtime stays in NC AGI. It also left out the $4,500 NC child deduction, so it taxed $49,055 instead of $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model added the overtime premium on top of wages, which already include it, and did not subtract the 401(k) and IRA contributions. It also left out the $4,500 child deduction and used a 4.5% rate instead of the 2026 rate of 3.99%."
-us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied a federal standard deduction instead of NC's $25,500 standard deduction and $4,500 child deduction, and it used a 4.5% rate instead of 3.99%. It also subtracted $19.41 of nonrefundable credits that do not exist."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the wrong tier of the NC child deduction: $500 per child instead of the $1,500 that applies to MFJ AGI of $80,001–$100,000. That gave a $1,500 total instead of $4,500 and taxable income of $55,301 instead of $52,301. It then submitted $2,882, which does not match its own figure of $2,206.51."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated federal taxable income as NC's starting point and subtracted a $29,200 federal standard deduction. The correct path starts from federal AGI and subtracts the $25,500 NC standard deduction plus the $4,500 NC child deduction. It also used a 4.75% rate instead of 3.99% and invented a dependent credit. Its final $5,238 has no connection to any of its own steps."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed NC deductions and exemptions wipe out this household's tax, but NC has no personal exemptions. The $25,500 standard deduction and $4,500 child deduction leave $52,300.52 taxable at 3.99%, so the tax is $2,086.79, not zero."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left the $130 traditional IRA deduction out of AGI ($82,431) and used $2,500 per child for the NC child deduction instead of the $1,500 tier for MFJ AGI of $80,001–$100,000. It then switched between 4.25% and 3.99% and submitted $3,041, a number none of its calculations produce."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the NC child deduction entirely ($1,500 × 3 = $4,500) and also skipped the $130 traditional IRA deduction. As a result it taxed $56,931 instead of $52,300.52 at 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used $21,500 as the NC MFJ standard deduction instead of $25,500. It also applied the top child-deduction tier of $3,000 per child, which only covers MFJ AGI up to $40,000; the correct tier at $82,301 is $1,500 per child. The two errors partly offset each other, giving $30,500 of deductions instead of $30,000."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 FLSA overtime premium to wages that already include it, which inflated AGI to $90,047. It also left out the NC child deduction and used 4.25% instead of the 2026 rate of 3.99%. It then raised the result to $3,350 for an unspecified 'addback'."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It placed AGI of $82,301 in the right $80,001–$100,000 bracket but assigned it $1,000 per child instead of $1,500. That made the NC child deduction $3,000 instead of $4,500, so it taxed $53,801 instead of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $7,746 overtime premium twice, on top of gross wages that already include it, which gave AGI of $90,047. It also left out the $4,500 NC child deduction, so it taxed $64,547 instead of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI and standard deduction were right, but it left out the NC child deduction of $1,500 for each of three children. That left $56,801 taxable instead of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It left out the $4,500 NC child deduction, so it taxed $56,801 instead of $52,300.52. It also used a 4.0% rate instead of the 2026 statutory rate of 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the $25,500 standard deduction from AGI. It skipped the NC child deduction of $1,500 per child ($4,500), which overstated taxable income by $4,500."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's wages, which cut AGI to $61,093. The head has no wages, so those premiums do not reduce AGI. Starting from that low AGI, it chose the $2,000-per-child tier instead of the $1,500 tier that applies at the correct AGI of $82,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. At 3.99%, its $3,538 implies about $88,700 of taxable income, which means it skipped the $25,500 NC standard deduction and the $4,500 child deduction that bring taxable income down to $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It treated the head's $21,208 ESI premiums as a reduction to the spouse's wages, which produced AGI of $61,093. The head has no wages, so the premiums do not reduce AGI. The low AGI also pushed it into the $2,000-per-child tier instead of the correct $1,500 tier."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,538.77 implies taxable income near $88,700 at 3.99%, which means it skipped the $30,000 of NC standard and child deductions applied to $82,300.52 of AGI."
+us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a 3.74% NC rate instead of the 2026 statutory rate of 3.99%. It also subtracted a $130 spousal IRA contribution for the head that the inputs do not list, and it left out the $4,500 NC child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used $4,000 per child for the NC child deduction. The correct amount for MFJ AGI of $80,001–$100,000 is $1,500 per child, so it deducted $12,000 instead of $4,500 and understated taxable income at $44,801."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions eliminate NC taxable income, but the $25,500 standard deduction and $4,500 child deduction total only $30,000. That leaves $52,300.52 taxable at 3.99%, or $2,086.79."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no real derivation. At 3.99%, its $4,875 implies about $122,000 of taxable income, far more than the household's $82,300.52 AGI before any deductions. The figure ignores the $30,000 of NC deductions and the actual income level."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,957.29 equals 3.99% of $49,055. That reflects subtracting the $7,746 overtime premium to get AGI of $74,555 even though overtime does not reduce AGI, and leaving out the $4,500 NC child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $2,027 equals 3.99% of $50,801, which means it applied a $2,000-per-child deduction ($6,000). That is the tier for MFJ AGI of $60,001–$80,000; the correct amount at AGI of $82,300.52 is $1,500 per child ($4,500)."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the NC child deduction ($4,500) and the $130 traditional IRA deduction, so it taxed about $56,931 instead of $52,300.52 at 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used $1,000 per child ($3,000) for the NC child deduction instead of the $1,500 per child ($4,500) that applies to MFJ AGI of $80,001–$100,000. That overstated taxable income by $1,500, at $53,801."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied a flat rate to roughly $85k of gross wages after only the standard deduction. It ignored the 401(k) and IRA reductions to AGI and left out the $4,500 NC child deduction, and its $2,780 is higher than 3.99% of any reasonable base."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted only the $25,500 standard deduction from AGI of $82,301 and left out the NC child deduction of $1,500 for each of three children. It taxed $56,801 instead of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 FLSA overtime premium to reach AGI of $74,555, even though the overtime deduction does not lower the AGI that NC starts from. It also left out the $4,500 NC child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It removed the $7,746 overtime premium from AGI ($74,555), but NC starts from federal AGI of $82,300.52. That understated AGI pushed it into the $2,000-per-child tier instead of the correct $1,500-per-child tier."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted the $7,746 overtime premium twice, which gave AGI of $90,047. It also left out the $4,500 NC child deduction and used 4.25% instead of the 2026 rate of 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the overtime premium to reach AGI of $74,555 instead of $82,300.52. It then used $2,500 per child, which does not match either its own AGI tier ($2,000) or the correct tier ($1,500 per child, $4,500 total)."
+us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used gross wages of $85,209 as federal AGI, leaving out the traditional 401(k) ($2,778) and IRA ($130) reductions. It also left out the $4,500 NC child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It assumed personal exemptions for a family of five reduce the tax to about zero, but NC has no personal exemptions. Only the $25,500 standard deduction and $4,500 child deduction apply, which leaves $52,300.52 taxable."
+us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 overtime premium from AGI ($74,555 instead of $82,300.52) and left out the NC child deduction of $1,500 per child."
+us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the overtime premium on top of wages that already include it ($92,955), ignored the 401(k) and IRA reductions, and left out the $4,500 NC child deduction. It also used a 4.5% rate instead of 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reached taxable income of $81,509 without applying the $25,500 NC standard deduction or the $4,500 NC child deduction. It also used a 4.5% rate instead of 3.99% and subtracted a $19.41 nonrefundable credit that does not exist."
us,scenario_026,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output, violating the required structured-output contract."
us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly derived zero taxable income, zero gross tax, and no nonrefundable credits, but submitted -3195 instead of the zero produced by its calculation. Federal income tax before refundable credits does not become negative here, and its explanation explicitly ends with a value inconsistent with its own derivation."
@@ -2256,51 +2333,52 @@ us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F
us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly noted CT BBCE at 200% FPL with no asset test. It then still applied the 100% FPL net income test and ended at zero. It missed that BBCE households are categorically eligible, so they skip the net income test, and that a categorically eligible two-person household gets the $24/month minimum allotment ($288/yr) when the formula benefit is zero."
-us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income of about $2,308 exceeds 100% FPL. Connecticut's BBCE confers categorical eligibility, which bypasses the net income test. A categorically eligible two-person household receives the $24/month minimum allotment, totaling $288."
-us,scenario_027,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said $2,513/month exceeds a roughly $2,900 limit, which contradicts its own numbers. It also applied a $3,500 asset limit, but Connecticut's BBCE removes the asset test. The household is categorically eligible at 1.44× FPL and gets the $24/month two-person minimum allotment ($288/yr)."
-us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model denied SNAP because $48,000 exceeds a $4,500 elderly/disabled resource limit. Connecticut's BBCE through TANF non-cash benefits waives that limit. The categorically eligible two-person household gets the $24/month minimum allotment, $288 a year."
-us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model applied the federal $4,500 elderly/disabled resource limit and found the household ineligible. Connecticut's BBCE eliminates the asset test, so the household is categorically eligible. Because 30% of net income exceeds the maximum, it receives the $24/month minimum for a two-person household ($288/yr)."
-us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model concluded that income over the net income limit means no benefit. Under Connecticut BBCE the household is categorically eligible and skips the net income test. Categorically eligible households of one or two people get the $24/month minimum even when the formula benefit is zero, giving $288."
-us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model failed the household on the roughly $1,763 net income limit. It missed that Connecticut BBCE confers categorical eligibility, which waives the net income test. It also missed the guaranteed $24/month minimum allotment for categorically eligible two-person households ($288/yr)."
-us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model found the gross test passed at 200% FPL but then applied a $4,500 asset limit anyway. Connecticut's BBCE eliminates the resource test. The household is categorically eligible and receives the $24/month two-person minimum allotment ($288/yr)."
-us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled resource limit, which Connecticut's BBCE waives. With categorical eligibility and 30% of net income above the $546 maximum, the household receives the $24/month minimum allotment, $288 a year."
-us,scenario_027,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the $650 annual medical and OTC expenses as monthly and deducted about $615/month, which understated net income at $1,712. The expenses average only about $54/month, and the engine's net income is $2,118. At that level 30% of net income ($636) exceeds the $546 maximum, so only the $24/month minimum ($288/yr) applies, not $388."
-us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model denied eligibility because $48,000 exceeds a $4,250 elderly/disabled asset limit. Connecticut's BBCE waives the asset test. The categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model correctly passed the 200% FPL gross test but then applied the 100% FPL net income test. BBCE categorical eligibility waives that test. The model also never applied the $24/month minimum allotment that categorically eligible two-person households receive, which gives $288."
-us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model stopped at zero because net income exceeds the limit. Under Connecticut BBCE the household is categorically eligible, so the net income test is waived. A categorically eligible one- or two-person household receives the $24/month minimum allotment, $288 a year."
-us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit of $1,762.50 as disqualifying. Connecticut BBCE categorical eligibility bypasses that test. The formula's zero benefit is replaced by the $24/month two-person minimum allotment ($288/yr)."
-us,scenario_027,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model correctly identified BBCE eligibility and a zero formula benefit, then stopped. It omitted the rule that categorically eligible households of one or two people receive the minimum allotment ($24/month in FY2026) even when max allotment minus 30% of net income is zero or negative. That rule gives $288."
-us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a generic SNAP resource limit to the $48,000 in assets. Connecticut's BBCE removes the asset test, and the categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model recognized categorical eligibility and that 30% of net income exceeds the maximum allotment. It then gave zero instead of applying the minimum allotment rule for categorically eligible one- and two-person households, which is $24/month or $288 a year."
-us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied the minimum allotment for a categorically eligible two-person household, but it used the outdated $23/month figure. For FY2026 the minimum is $24/month (8% of the $298 one-person maximum, rounded up), so the correct total is $288, not $276."
-us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted a zero benefit from income and assets. Connecticut's BBCE waives the asset and net income tests, and categorically eligible two-person households receive the $24/month minimum allotment even when the formula gives zero, totaling $288."
-us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum allotment for a two-person household, but it used the prior-year $23/month minimum. The FY2026 minimum is $24/month, so the annual total is $288, not $276."
-us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the minimum allotment pathway correctly but used $23/month instead of the FY2026 value of $24/month (8% of the $298 one-person maximum, rounded up). That understates the annual amount by $12."
-us,scenario_027,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,The model applied an asset limit that Connecticut's BBCE waives. It also treated a zero formula benefit as final and missed the $24/month minimum allotment guaranteed to categorically eligible two-person households ($288/yr).
-us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable SNAP value or explanation, so no substantive answer was given for the $288 reference."
-us,scenario_027,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum allotment, then returned zero. Categorically eligible households of one or two people under Connecticut BBCE receive the $24/month minimum allotment, $288 a year."
-us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model applied typical asset and income limits to deny eligibility. Connecticut's BBCE waives the asset test and confers categorical eligibility at 1.44× FPL, so the two-person household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of shelter costs as grounds for a zero estimate. It never computed categorical eligibility under Connecticut BBCE or applied the $24/month minimum allotment for categorically eligible two-person households, which yields $288."
-us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model stopped at net income being above the limit and the benefit phasing out. It missed that Connecticut BBCE confers categorical eligibility, and that a categorically eligible two-person household with a zero formula benefit receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model concluded the benefit formula produces no allotment and returned zero. It missed the minimum allotment rule for categorically eligible one- and two-person households, which gives $24/month and $288 annually."
-us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model applied the two-person net income limit as disqualifying. Under Connecticut BBCE the household is categorically eligible, so the net income test is waived and the $24/month minimum allotment applies, totaling $288."
-us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the elderly/disabled SNAP resource limit to the $48,000 in bank assets. Connecticut's BBCE eliminates the asset test, and the categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model found no positive formula allotment and returned zero. It omitted the rule that categorically eligible households of one or two people receive the minimum allotment ($24/month in FY2026), which yields $288."
-us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model denied the benefit because income exceeds the two-person net income limit. Connecticut BBCE categorical eligibility waives the net income test, and the household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum, then gave zero. It missed that categorically eligible one- and two-person households are guaranteed the $24/month minimum allotment, $288 a year."
-us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model cited typical SNAP asset limits, which Connecticut's BBCE waives. Gross income at 1.44× FPL confers categorical eligibility, so the two-person household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model applied a $4,500 elderly/disabled resource limit and called the household ineligible regardless of income. Connecticut's BBCE eliminates the asset test, and the household receives the $24/month two-person minimum allotment ($288/yr)."
-us,scenario_027,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum allotment. It returned zero instead of the $24/month minimum allotment guaranteed to categorically eligible one- and two-person households, which is $288."
-us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"The model correctly applied BBCE, the asset waiver, and the minimum allotment, but it used the outdated $23/month minimum. The FY2026 minimum is $24/month, so the annual total is $288, not $276."
-us,scenario_027,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model compared annual net income directly to the annual maximum allotment instead of subtracting 30% of monthly net income. It also never applied the $24/month minimum allotment for categorically eligible two-person households, which yields $288."
-us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model failed the household on the 100% FPL net income test (about $1,762/month). Connecticut BBCE categorical eligibility waives that test. The two-person household then receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income is above 100% FPL. Under Connecticut BBCE the household is categorically eligible and skips the net income test, so the $24/month minimum allotment applies, totaling $288."
-us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the two-person net income limit as disqualifying. It missed that Connecticut BBCE makes the household categorically eligible, and that such one- and two-person households receive the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used a wrong 200% FPL gross limit of about $30,120 a year. The actual limit for two people is roughly $42,300, and the household's $30,435 is only 1.44× the poverty guideline. The model also applied a $4,250 asset limit that Connecticut's BBCE waives, so it missed the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the net income test and the 30% benefit reduction to reach zero. Connecticut BBCE categorical eligibility waives the net income test, and categorically eligible two-person households receive the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model claimed income exceeds the SNAP gross limit, but $2,536/month is 1.44× FPL, well under Connecticut's 200% BBCE threshold. It also applied an asset limit that BBCE waives. The categorically eligible household receives the $24/month minimum allotment ($288/yr)."
-us,scenario_027,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $51,000 resource limit and asserted that net income is below the poverty guideline, which led to a $430/month benefit. Net income is actually $2,118/month, so 30% of it ($636) exceeds the $546 maximum. Only the $24/month minimum allotment applies, totaling $288."
+us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly saw that CT BBCE removes the asset test, but it still applied the 100% FPL net income test and denied the household. BBCE categorical eligibility also exempts the household from the net income test. As a categorically eligible two-person household, it gets the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model applied the regular elderly/disabled 100% FPL net income test (about $1,763) and concluded 'ineligible'. It never applied Connecticut's TANF non-cash BBCE, which removes that test. A categorically eligible two-person household with a $0 formula benefit receives the $24/month minimum allotment."
+us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model claimed $2,513/month exceeds an approximately $2,900 limit, which contradicts its own numbers. It also applied a $3,500 elderly/disabled asset limit. Under Connecticut BBCE, gross income at 1.44× poverty is categorically eligible with no asset test, so the household receives the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model denied SNAP based on the $48,000 in bank assets. Connecticut BBCE through TANF non-cash eligibility waives the asset test. The household is categorically eligible and gets the $24/month two-person minimum allotment."
+us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model applied a roughly $4,500 elderly/disabled resource limit to the $48,000 in assets. Connecticut's BBCE waives the SNAP asset test for this household, whose gross income is 1.44× poverty. As a result, the household qualifies for the $24/month minimum allotment."
+us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model treated the roughly $4,500 elderly/disabled resource limit as binding. Connecticut BBCE categorical eligibility removes the asset test. The $0 formula benefit (30% × $2,118 = $636 > $546) is replaced by the $24/month minimum allotment for two-person households."
+us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,The model denied benefits because income exceeds the two-person net income limit. That test doesn't apply to a household that is categorically eligible through Connecticut's TANF non-cash BBCE. Such a household with a $0 formula benefit receives the $24/month minimum allotment ($288/year).
+us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model compared net income of about $2,308 to the roughly $1,763 net income limit and stopped there. Connecticut BBCE makes the household categorically eligible, which exempts it from the net income test. The model also omitted the two-person minimum allotment of $24/month."
+us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model correctly found that gross income passes. It then applied a $4,500 elderly/disabled asset limit and denied on assets. Connecticut BBCE through TANF non-cash eligibility waives the asset test, so the household gets the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled resource limit and called the household ineligible. Connecticut BBCE categorical eligibility removes the asset test entirely. A categorically eligible two-person household receives at least the $24/month minimum allotment."
+us,scenario_027,snap,claude-sonnet-5.5,llm_error,period_annualization,False,"The model treated the $650 annual medical and OTC expenses as a monthly amount. That produced a $615/month excess medical deduction instead of about $19 and pushed net income down to about $1,712. With the correct deductions, 30% of net income ($636) exceeds the $546 maximum, so only the $24/month minimum allotment applies, for $288/year."
+us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled asset limit to the $48,000 in bank assets. Connecticut BBCE waives the asset test for this household, whose gross income is 1.44× poverty. The household is categorically eligible for the $24/month minimum allotment."
+us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model saw that gross income passes 200% FPL but then applied a 100% FPL net income test (about $1,703) as disqualifying. BBCE categorical eligibility exempts the household from the net income test. A $0 formula benefit for a categorically eligible two-person household is replaced by the $24/month minimum."
+us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model concluded that net income exceeds the two-person net income limit. It never recognized Connecticut's TANF non-cash BBCE, which makes the household categorically eligible and exempt from that test. The household therefore receives the $24/month minimum allotment."
+us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test ($1,762.50) to net income of about $2,313 and returned $0. Connecticut BBCE categorical eligibility removes the net income test. The two-person minimum allotment of $24/month then applies, for $288/year."
+us,scenario_027,snap,gemini-3-flash-preview,llm_error,other,False,"The model correctly found BBCE eligibility and a $0 formula benefit (30% of net income exceeds the $546 maximum). It then left out the SNAP minimum allotment. Eligible one- and two-person households receive $24/month (8% of the $298 one-person maximum), or $288/year."
+us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model denied SNAP based on the $48,000 in assets. Connecticut BBCE through TANF non-cash eligibility waives the resource test. The household is categorically eligible for the $24/month two-person minimum allotment."
+us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,other,False,"The model recognized categorical eligibility and the $0 formula benefit, then returned $0. Categorically eligible one- and two-person households are guaranteed the minimum allotment of $24/month in FY2026, which totals $288/year."
+us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum-allotment rule but used the FY2025 minimum of $23. For 2026 the minimum is 8% of the $298 one-person maximum allotment, which rounds to $24/month, giving $288/year rather than $276."
+us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave no derivation, and its $0 matches either denying on the asset or income tests or stopping at the $0 formula benefit. Connecticut BBCE makes the household categorically eligible regardless of the $48,000 in assets. The model also omitted the $24/month minimum allotment for two-person households."
+us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly found BBCE categorical eligibility and applied the minimum allotment, but at the FY2025 value of $23/month. The 2026 minimum is $24/month (8% of the $298 one-person maximum), so the annual total is $288, not $276."
+us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $23/month minimum allotment, which is the FY2025 figure. The 2026 minimum is 8% of the $298 one-person maximum, rounded to $24/month, which gives $288 annually."
+us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"The model treated the $48,000 in assets as exceeding SNAP limits and the formula zero as final. Connecticut BBCE waives the asset test. Categorically eligible two-person households receive the $24/month minimum allotment even when 30% of net income exceeds the $546 maximum."
+us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no SNAP value or explanation, so there was no answer to score."
+us,scenario_027,snap,glm-5.3,llm_error,other,False,"The model correctly computed that 30% of net income exceeds the two-person maximum allotment, but it stopped at a $0 benefit. The household is categorically eligible through Connecticut BBCE, so it receives the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model denied SNAP on the $48,000 in assets and on income under 'typical' rules. Connecticut BBCE through TANF non-cash eligibility waives the asset test, and gross income at 1.44× poverty qualifies. The household gets the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to $0 because shelter and rent inputs were missing. It never worked out that Connecticut BBCE makes the household categorically eligible. Under that eligibility, the $0 formula benefit is replaced by the $24/month two-person minimum allotment."
+us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model said net income is above the SNAP limit for an elderly/disabled two-person household and returned $0. Connecticut BBCE categorical eligibility exempts the household from the net income test. The model also missed the $24/month minimum allotment.
+us,scenario_027,snap,gpt-5.6-luna,llm_error,other,False,"The model stopped at the $0 formula benefit after standard and medical deductions. A categorically eligible two-person household receives the $24/month minimum allotment, which totals $288/year."
+us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model denied SNAP because net income is above the two-person net income limit. Connecticut BBCE through TANF non-cash eligibility makes the household categorically eligible, which removes that test. The household then receives the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the elderly/disabled resource limit to the $48,000 in assets. Connecticut BBCE waives the asset test. The household is categorically eligible for the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,gpt-6-luna,llm_error,other,False,"The model correctly found no positive formula allotment, but it treated $0 as final. Categorically eligible two-person households get the minimum allotment of $24/month (8% of the $298 one-person maximum), which totals $288/year."
+us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,The model applied the two-person net income limit as disqualifying. Under Connecticut's TANF non-cash BBCE the household is categorically eligible and exempt from that test. Its benefit is the $24/month minimum allotment.
+us,scenario_027,snap,gpt-6.1-sol,llm_error,other,False,"The model correctly found that the benefit reduction (30% × $2,118 = $636) exceeds the $546 maximum, then stopped at $0. It omitted the minimum allotment, which gives a categorically eligible two-person household $24/month."
+us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model treated the $48,000 in assets as exceeding SNAP limits. Connecticut BBCE removes the asset test, and gross income at 1.44× poverty qualifies. The household is categorically eligible for the $24/month minimum allotment."
+us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model applied an approximately $4,500 elderly/disabled resource limit 'regardless of income'. Connecticut BBCE categorical eligibility waives the resource test. This two-person household therefore receives the $24/month minimum allotment."
+us,scenario_027,snap,grok-4.6,llm_error,other,False,"The model correctly found that 30% of net income exceeds the two-person maximum, then returned $0. It omitted the minimum allotment of $24/month that categorically eligible two-person households receive."
+us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum-allotment rule but used the FY2025 minimum of $23/month. The 2026 minimum is 8% of the $298 one-person maximum, rounded to $24/month, giving $288/year."
+us,scenario_027,snap,grok-build-0.1,llm_error,other,False,"The model compared annual net income (about $30,000) directly with the annual maximum allotment instead of subtracting 30% of net income from the maximum. It also omitted the $24/month minimum allotment that applies to a categorically eligible two-person household."
+us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test (about $1,762) to net income of about $2,317 and called the household ineligible. Connecticut BBCE categorical eligibility removes that test. The household receives the $24/month minimum allotment."
+us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model applied the 100% FPL net income limit as disqualifying because there are no shelter costs. Under Connecticut's TANF non-cash BBCE the household is categorically eligible and exempt from the net income test. Its $0 formula benefit becomes the $24/month two-person minimum.
+us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model denied SNAP because net income exceeds the two-person net income limit. Connecticut BBCE makes the household categorically eligible, which waives that test. The model also missed the $24/month minimum allotment."
+us,scenario_027,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model used about $30,120 as the 200% FPL gross limit for two people; the real limit is about $42,300/year, and $30,435 is only 1.44× poverty. It also applied a $4,250 asset limit, which Connecticut BBCE waives. The household is categorically eligible for the $24/month minimum allotment."
+us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the poverty-guideline net income test and the formula zero, then returned $0. Connecticut BBCE categorical eligibility exempts the household from the net income test. Two-person categorically eligible households receive the $24/month minimum allotment."
+us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model claimed that $2,536/month exceeds the gross income limit, but that is 1.44× poverty and under Connecticut's BBCE limit. It also applied a resource limit that BBCE waives. The household is categorically eligible for the $24/month minimum allotment."
+us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $51,000 resource limit and said net income is below the poverty guideline, when net income of $2,118 is 1.2× poverty. With 30% × $2,118 = $636 exceeding the $546 maximum, the formula benefit is $0, and only the $24/month minimum allotment applies. The correct total is $288/year, not $5,160."
us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated Medicaid MAGI as counting only taxable Social Security and put countable income at about $275. Medicaid MAGI adds back non-taxable Social Security benefits, so household MAGI is the full $30,435, or 1.41x FPL. That is above the 138% expansion limit, and the model itself noted that $30,435 exceeded its own ~$28,850 threshold."
us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It treated blindness and disability as categorical Medicaid eligibility and looked only at the spouse's own zero income. It also asserted that the $48,000 in assets satisfies the ABD test. The spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits ($3,000 resource limit), so no disability category applies. The adult MAGI path also fails at 1.41x FPL against the 138% limit."
@@ -2315,17 +2393,18 @@ us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_
us,scenario_027,spouse_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It set MAGI at $275 by excluding Social Security. Medicaid MAGI adds back non-taxable Social Security benefits, so household MAGI is $30,435, or 1.41x FPL, which exceeds Connecticut's 138% expansion limit."
us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It computed HUSKY D household MAGI as $275 (interest only) because it did not count Social Security. Medicaid MAGI adds back non-taxable Social Security benefits, giving $30,435, which even exceeds its own $28,207 figure for 138% FPL. The engine puts it at 1.41x FPL, so the expansion test fails."
us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It granted SSI-related Medicaid because the spouse is blind, disabled and has no income of her own. It ignored that the spouse receives $0 SSI: SSI counts the couple's combined $30,435 income and $48,000 in resources, both far above the couple limits. The MAGI adult path also fails at 1.41x FPL against 138%."
-us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disability flag as enough for Medicare. It noted that disability Medicare 'typically' follows SSDI but never checked whether the spouse receives SSDI. The spouse has $0 of their own Social Security disability income, so there is no 24-month SSDI entitlement and no under-65 pathway."
-us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model claimed that disability status confers Medicare regardless of age. In fact, under-65 Medicare requires 24 months of the person's own SSDI entitlement (or ESRD/ALS). The spouse has no SSDI income; only the head does."
-us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly stated that under-65 Medicare requires having received disability benefits, then assumed the spouse receives them because of the disability flag. The spouse has no Social Security disability income listed, so the 24-month SSDI entitlement test fails."
-us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model assumed the spouse receives disability benefits based only on the disability flag. The listed SSDI income belongs to the head, and the 39-year-old spouse has no SSDI entitlement of their own, so they meet neither the age-65 test nor the 24-month SSDI test."
-us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model gave the spouse Medicare through the head's SSDI receipt, treating the spouse as the 'disabled spouse of a Social Security beneficiary.' Medicare's disability pathway requires the individual's own 24 months of disability-benefit entitlement. The spouse has $0 of their own SSDI, so they are not eligible."
-us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted that the spouse has received Social Security disability benefits for 24 months, which contradicts the household facts. The only SSDI income ($30,160) belongs to the head, and the spouse's unlisted SSDI must be treated as $0."
-us,scenario_027,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for spouse_medicare_eligible. The correct answer is 0: the 39-year-old spouse is under 65 and has no SSDI entitlement of their own, so they fail both the age test and the 24-month disability test."
-us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated a 'qualifying disability' as meaning the disability flag itself. PolicyEngine's disability pathway requires 24 months of Social Security disability benefit receipt, and the spouse has none."
-us,scenario_027,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a 'disability-based Medicare eligibility rule' as if the disability flag triggered it. That rule is keyed to 24 months of the person's own SSDI receipt, and the spouse's SSDI is $0."
-us,scenario_027,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the explicit disability flag as meeting the Medicare disability criterion. That criterion requires 24 months of the individual's own Social Security disability benefit receipt, which the spouse does not have, so an under-65 spouse is not eligible."
-us,scenario_027,spouse_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model treated the blindness and disability flags as a Medicare pathway. Medicare's under-65 route requires 24 months of the person's own SSDI entitlement (or ESRD/ALS), not blindness or disability status alone. The spouse has no SSDI income."
+us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model noted that disability-based Medicare normally follows SSDI receipt, then made the spouse eligible on the disability flag alone. The spouse has no Social Security disability income, so the 24-month SSDI entitlement path is never met, and at age 39 the spouse fails the age-65 test."
+us,scenario_027,spouse_medicare_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model's own reasoning was right: disability-based Medicare requires SSDI receipt, the spouse has none, so the spouse is not eligible. It still submitted 1, which contradicts its own conclusion; the answer should have been 0."
+us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model said disability status gives Medicare eligibility regardless of age. Under-65 Medicare requires 24 months of the person's own SSDI entitlement, and the spouse receives no Social Security disability income, so the spouse is not eligible."
+us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly stated that under-65 Medicare requires prior receipt of disability benefits, then assumed the spouse receives them because they are disabled. The spouse's Social Security disability income is $0, so the SSDI-based path fails, and at age 39 the spouse fails the age test."
+us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated the spouse as 'receiving disability-related status' and made them Medicare-eligible from the disability flag. Medicare's disability pathway requires 24 months of the person's own SSDI benefits, and the spouse has none."
+us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model credited the head's $30,160 SSDI to the spouse, claiming a disabled spouse of a Social Security beneficiary qualifies for Medicare. Disability-based Medicare requires the person's own 24 months of SSDI (or disabled widow(er)/adult-child) entitlement. The spouse receives no disability benefits, so they are ineligible at age 39."
+us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented 24 months of Social Security disability benefits for the spouse, even though the prompt lists no SSDI income for them (only the head has SSDI). Without the spouse's own SSDI, and at age 39, no Medicare pathway applies."
+us,scenario_027,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for spouse_medicare_eligible, so its answer was missing rather than wrong on substance. The correct answer is 0: the spouse is 39 and has no own SSDI receipt."
+us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model claimed PolicyEngine counts the disability flag as a Medicare qualification. PolicyEngine's under-65 path instead requires 24 months of Social Security disability benefit receipt (or ESRD/ALS), and the spouse has $0 SSDI, so the answer should have been 0."
+us,scenario_027,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a 'disability-based Medicare rule' using only the spouse's disabled flag. That rule requires 24 months of the person's own Social Security disability benefits, which the 39-year-old spouse does not receive, so the spouse is not eligible."
+us,scenario_027,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated 'explicitly disabled' as enough for Medicare. The disability criterion turns on 24 months of the person's own SSDI receipt, and the spouse has zero Social Security disability income, so they are not eligible at age 39."
+us,scenario_027,spouse_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model concluded that the blind and disabled flags give Medicare eligibility through the disability route. That route requires 24 months of the person's own Social Security disability benefits, and the spouse receives none; blindness and disability without SSDI lead to SSI/Medicaid, not Medicare."
us,scenario_027,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -2428,108 +2507,111 @@ us,scenario_028,child3_medicare_eligible,glm-5.2,parse_contract_failure,missing_
us,scenario_028,child3_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child3_wic_eligible, violating the required structured-output contract."
us,scenario_028,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age range, although WIC child eligibility ends at age five. It also failed to apply the household income test, under which approximately $61,277 exceeds the 185%-of-poverty limit."
us,scenario_028,child3_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied both controlling conditions: a 10-year-old is outside WIC's under-five child category, and household income of approximately $61,277 exceeds the applicable WIC income limit. Its statements that the child met the age limit and the household fell within the income range are both incorrect."
-us,scenario_028,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model added the $1,267 of child support to AGI even though child support received is excluded from gross income, and used a $23,500 single-filer standard deduction rather than the $24,150 head-of-household amount. It then applied only half of its own $6,000 CTC ($2,290) against a $4,574 liability with no rule justifying the split; the nonrefundable CTC covers the entire pre-credit tax, leaving $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own reasoning reached the correct conclusion — $6,000 of CTC 'fully offsets tax to 0' against its ~$4,077 liability — and then submitted $2,818, a number that appears nowhere in its derivation. It abandoned a correct nonrefundable-CTC application at the final step; with $6,600 of credit against $3,949.20 of tax, the answer is $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model inverted the CTC ordering, computing the refundable ACTC first at the full $1,700 × 3 = $5,100 cap and treating only the leftover as available nonrefundably, which left $1,596 of tax standing. The nonrefundable CTC is applied first and absorbs the entire $3,949.20 liability; the ACTC is the $2,650.80 residual of the $6,600 credit, not a prior claim against it."
-us,scenario_028,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model derived '$4,059 - $4,059 = $0' by correctly applying the nonrefundable CTC against its computed liability, then overrode that with $2,114 for unspecified 'nonrefundable credit ordering and rounding' nuances. No ordering rule reduces the nonrefundable CTC below the $3,949.20 liability when the total credit is $6,600, so the answer is $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a lapsed $1,000-per-child CTC ($3,000 total) for 2026 while simultaneously using a post-TCJA-style $23,130 standard deduction, mixing pre- and post-sunset parameter regimes. The 2026 credit is $2,200 per child, so $6,600 exceeds the $3,949.20 liability and no tax remains before refundable credits."
-us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model priced 2026 under pre-TCJA law: a $12,150 standard deduction plus $21,200 of personal exemptions and a $1,000-per-child CTC. Personal exemptions are repealed for 2026, the head-of-household standard deduction is $24,150, and the CTC is $2,200 per child, so $6,600 of credit wipes out the $3,949.20 of tax."
-us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model used a $29,200 standard deduction — a married-filing-jointly figure, not the $24,150 head-of-household amount — and then reported $1,395 of residual tax after subtracting $6,000 of CTC from a liability its own $30,810 of taxable income puts near $3,340. The credit exceeds the liability under either deduction, so the nonrefundable CTC leaves $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly assumed TCJA expiration for 2026, applying a ~$12,000 head-of-household standard deduction, $5,350 personal exemptions for four people, and a $1,000-per-child CTC. For 2026 the head-of-household standard deduction is $24,150, personal exemptions remain repealed, and the CTC is $2,200 per child, so $6,600 of credit fully offsets the $3,949.20 liability."
-us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied post-TCJA-expiration parameters — a ~$12,350 standard deduction plus $21,400 of personal exemptions and a $1,000-per-child CTC — to reach $39 of residual tax. Personal exemptions are repealed for 2026, the head-of-household standard deduction is $24,150, and the CTC is $2,200 per child, producing $3,949.20 of tax fully absorbed by $6,600 of nonrefundable credit."
-us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model gave no derivation, and $491.50 is consistent with subtracting only about $3,000 of CTC — the lapsed $1,000-per-child amount — from a roughly $3,500 liability. The 2026 credit is $2,200 per child, so $6,600 of nonrefundable CTC exceeds the $3,949.20 of tax on $35,860 of taxable income and the answer is $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so the key was absent from the submitted outputs rather than substantively wrong. Nothing in its response engages the $3,949.20 liability or the $6,600 of nonrefundable CTC that reduces it to $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model got AGI ($60,010), the $24,150 standard deduction, taxable income ($35,860), and the $2,200-per-child credit right, then carved a fixed '$1,800 per child refundable' slice out of the $6,600 first and applied only the $1,200 remainder nonrefundably. The refundable ACTC is the residual left after the nonrefundable CTC offsets the full $3,949.20 liability — here $2,650.80 — and its per-child cap is $1,700, not $1,800."
-us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model asserted that the standard deduction and child tax credit reduce tax 'to about $3,000' without ever computing the credit offset. Tax on $35,860 of taxable income is $3,949.20 and the $6,600 nonrefundable CTC exceeds it entirely, so the correct figure is $0, not a residual near the pre-credit liability."
-us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model included the nontaxable $1,267 of child support in AGI, and $7,342 is consistent with applying roughly a flat 12% to the full $61,277 of household income with no standard deduction and no child tax credit. Its answer exceeds even the $3,949.20 pre-credit liability, which the $6,600 nonrefundable CTC reduces to $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model used a $22,500 standard deduction instead of the 2026 head-of-household $24,150, then stated that the $6,600 CTC needs only part of itself 'to reduce tax to zero' before submitting $1,956, contradicting its own derivation. With $6,600 of credit against $3,949.20 of tax, the nonrefundable CTC leaves nothing."
-us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model supplied no derivation, and $2,890 is consistent with applying only about $1,100 of child tax credit against a roughly $3,990 liability. The full $6,600 of 2026 CTC for three qualifying children is applied nonrefundably first and exceeds the $3,949.20 of tax, so the answer is $0."
-us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model explicitly assumed TCJA expiration for 2026 — a $12,286 head-of-household standard deduction, pre-2018 brackets with a 15% second rate, and a $1,000-per-child CTC. For 2026 the standard deduction is $24,150, the second bracket rate is 12%, and the CTC is $2,200 per child, giving $3,949.20 of tax fully offset by $6,600 of nonrefundable credit."
-us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for this variable, so the failure is a missing output rather than a computational error. The requested figure is $0: $6,600 of nonrefundable CTC exceeds the $3,949.20 of tax on $35,860 of taxable income."
-us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used the lapsed $2,000-per-child credit and reversed the ordering, subtracting an assumed $5,100 of refundable ACTC from the $6,000 total to leave only $900 of nonrefundable CTC against a $4,065 liability. The 2026 credit is $2,200 per child, and the nonrefundable portion is applied first against the entire $3,949.20 of tax, with the ACTC being only the $2,650.80 residual."
-us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Its own working got a total of about $2,916 (ACTC $2,665 plus EITC $251), but it submitted $5,100, the full $1,700-per-child refundable cap, which ignores the limit to the CTC left after offsetting tax. Its EITC inputs were also wrong: an $8,046 maximum and a $23,000 phase-out start in place of $8,231 and $23,890, which gave $251 instead of $624.13."
-us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The ACTC was about right ($2,644.80 against $2,650.80). The EITC phase-out, however, started at $23,000 instead of the 2026 threshold of $23,890 for non-joint filers with 3 children, which understated the EITC at $436.70 against $624.13."
-us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It counted nontaxable child support in income and denied the EITC entirely, even though $60,010 is below the 3-child phase-out end of about $62,974 (reference EITC $624.13). It then asserted a $3,850 ACTC without subtracting the $3,949.20 tax offset from the $6,600 CTC ($2,200 per child), which leaves $2,650.80."
-us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600), which gave a $2,037 ACTC instead of $2,650.80. It also zeroed the EITC by putting the phase-out end at about $59,899, when the credit actually runs to about $62,974 and is worth $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It overstated the pre-credit tax at about $4,800 (correct: $3,949.20) against a $6,000 CTC, worked out an ACTC of about $1,200, and then submitted an unsupported $4,500. It also zeroed the EITC with a phase-out end of about $59,899, which misses the $624.13 still available at $60,010."
-us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It paid the full $1,700 × 3 = $5,100 ACTC, missing that the ACTC is capped at the CTC left after offsetting tax: $6,600 minus $3,949.20 is $2,650.80. It also claimed no EITC, when $60,010 is still inside the 3-child phase-out range and yields $624.13."
-us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It combined a $2,000-per-child CTC ($6,000) with a $22,500 standard deduction and 2025-level brackets, which put tax at $4,170 and the ACTC at $1,830. The OBBBA figures are $2,200 per child and $24,150, giving tax of $3,949.20 and an ACTC of $2,650.80. It also placed the 3-child EITC phase-out end at about $58,000, which zeroes the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC and zeroed the EITC with an invented phase-out threshold of $56,838. The actual phase-out runs from $23,890 at 21.06% and leaves $624.13. The submitted $2,000 is a round guess, not the $6,600 − $3,949.20 = $2,650.80 unused-CTC ACTC."
-us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It took the ACTC as $6,000 − $4,170 = $1,830, using the pre-OBBBA $2,000-per-child CTC and an overstated tax. The correct figures are $6,600 − $3,949.20 = $2,650.80. It also estimated the EITC at $165 instead of $624.13 ($8,231 maximum, $23,890 phase-out start)."
-us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It claimed tax liability exceeds the CTC. In fact the $3,949.20 tax is well below the $6,600 CTC, which leaves a $2,650.80 refundable ACTC. It also claimed the EITC phases out by $60,000, when the 3-child EITC still pays $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The EITC was close ($662.87 against $624.13). It dropped the ACTC entirely by assuming the nonrefundable CTC was fully used, missing that the $6,600 CTC ($2,200 per child) exceeds the $3,949.20 tax by a refundable $2,650.80."
-us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) against a $4,073.20 tax, which gave a $1,926.80 ACTC instead of $6,600 − $3,949.20 = $2,650.80. Its EITC parameters were also wrong: an $8,271 maximum and a $24,024 phase-out start in place of $8,231 and $23,890, which gave $692.35 instead of $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $66 of leftover CTC implies a pre-credit tax of about $6,500, far above the correct $3,949.20 (taxable income of $35,860 after the $24,150 head-of-household standard deduction), so it lost most of the $2,650.80 ACTC. It also understated the EITC at $524 instead of $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It returned $0, missing both refundable components. The $6,600 CTC exceeds the $3,949.20 tax and leaves a $2,650.80 ACTC, and $60,010 is below the 3-child EITC phase-out end (about $62,974), leaving $624.13 of EITC."
-us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It set the refundable CTC to $0 on the claim that the CTC was fully used against tax. The $6,600 CTC ($2,200 per child) exceeds the $3,949.20 pre-credit tax, and the $2,650.80 difference is refundable within the $5,100 cap. Its EITC of $574 also fell short of $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, the pre-TCJA amount) against a $3,039 tax, when the 2026 CTC is $2,200 per child ($6,600) and exceeds the $3,949.20 tax by a refundable $2,650.80. It also denied the EITC, which pays $624.13 at $60,010 because the 3-child phase-out ends near $62,974."
-us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It returned $0 with only a generic phase-out rationale. It never computed the $2,650.80 of the $6,600 CTC left after the $3,949.20 tax offset, or the $624.13 EITC still available in the 3-child phase-out range."
-us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed the ACTC from a $6,000 CTC ($2,000 per child) less a $4,089.20 tax, giving $1,910.80, instead of the OBBBA $6,600 less $3,949.20 = $2,650.80. It also left out the $624.13 EITC entirely."
-us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It claimed the CTC was fully absorbed as nonrefundable and the EITC fully phased out, and both claims are false. The $6,600 CTC exceeds the $3,949.20 tax by a refundable $2,650.80, and the 3-child EITC still pays $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $1,838.80 ACTC implies a $6,000 CTC ($2,000 per child) less a $4,161.20 tax, when the correct figure is $6,600 − $3,949.20 = $2,650.80. Its EITC of $327.51 also understates the $624.13 that follows from an $8,231 maximum, a $23,890 phase-out start, and a 21.06% rate."
-us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to score against the $3,274.93 reference."
-us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It paid an invented $1,800 per child refundable amount ($5,400), ignoring the $1,700 cap and the rule that the ACTC cannot exceed the CTC left after offsetting tax ($6,600 − $3,949.20 = $2,650.80). It also zeroed the EITC, which pays $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated the entire $2,000-per-child CTC as refundable ($6,000). That misses both the offset against the $3,949.20 tax and the $1,700-per-child refundable cap, so the ACTC is $2,650.80. It also omitted the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It returned $0 and never computed the ACTC, which is the $2,650.80 of the $6,600 CTC left after the $3,949.20 tax. It also wrongly treated the 3-child EITC as fully phased out at $60,010, where $624.13 remains."
-us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but applied only $2,184.20 of pre-credit tax instead of $3,949.20 (tax on $35,860 of taxable income after the $24,150 standard deduction). That inflated the ACTC to about $4,416, and it then submitted an unexplained $4,444. It also dropped the $624.13 EITC by assuming full phase-out."
-us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The refundable CTC of $2,650.80 was correct, but it declared the EITC unavailable at this income. The 3-child EITC does not fully phase out until about $62,974 ($23,890 + $8,231 / 0.2106), so it still pays $624.13 at $60,010 AGI."
-us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It paid the full $1,700 × 3 = $5,100 refundable cap, missing that the ACTC is limited to the CTC left after the $3,949.20 tax offset, which is $2,650.80. It also omitted the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The ACTC was correct ($2,651), but the EITC was overstated at $650 instead of $624.13. The exact 2026 figure is $8,231 − 0.2106 × ($60,010 − $23,890), and its rough estimate of the 3-child phase-out missed it by about $26."
-us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It got the refundable CTC exactly right ($6,600 − $3,949.20 = $2,650.80) but left out the EITC. With 3 children, the EITC phases down from $8,231 at 21.06% above $23,890 and leaves $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It declared that no refundable credits qualify. That misses the $2,650.80 ACTC (the $6,600 CTC less $3,949.20 of tax) and the $624.13 EITC, which is still inside the 3-child phase-out range at $60,010."
-us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its remaining ACTC of about $40 implies a pre-credit tax of roughly $6,560 instead of $3,949.20, which wiped out most of the $2,650.80 ACTC. Its EITC used an $8,250 maximum and a phase-out start near $24,000 (instead of $8,231 and $23,890) and came out at $660 instead of $624.13."
-us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its unused CTC of $48 implies a pre-credit tax of about $6,552 against the $6,600 CTC, far above the correct $3,949.20, so it lost almost all of the $2,650.80 ACTC. Its EITC of $636 used a slightly wrong $8,239 maximum and $23,910 threshold instead of $8,231 and $23,890."
-us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,other,False,"It offset $3,948 of tax against the $6,600 CTC but miscalculated the remainder as $2,952 instead of $2,652, overstating the ACTC by $300. It also overstated the EITC at $661 instead of $624.13."
-us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the CTC reverts to $1,000 per child ($3,000 in total) and is fully absorbed by tax. OBBBA made $2,200 per child permanent, so the $6,600 CTC exceeds the $3,949.20 tax by a refundable $2,650.80. Its EITC also used reverted parameters ($8,302 maximum, $24,099 start) and gave $739 instead of $624.13."
-us,scenario_028,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ACTC was essentially right ($2,649 against $2,650.80). The EITC, however, came out at about $340 instead of $624.13; the $8,231 maximum, $23,890 phase-out start and 21.06% rate produce the correct figure."
-us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to score against the $3,274.93 reference."
-us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The EITC was exactly right at $624.13, but it put the pre-credit tax at $3,838.10 instead of $3,949.20. The correct figure is $1,770 plus 12% of the $35,860 taxable income above $17,700, after the $24,150 head-of-household standard deduction. The low tax inflated the ACTC to $2,761.90 instead of $2,650.80."
-us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It paid the full $5,100 ACTC cap without limiting it to the $6,600 CTC less the $3,949.20 tax ($2,650.80). It also zeroed the EITC despite stating a phase-out end of about $61,500, which is above the $60,010 AGI; the correct EITC is $624.13."
-us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The ACTC was correct at $2,650.80. The EITC phase-out, however, started at $22,210 instead of the 2026 non-joint threshold of $23,890, which cut the EITC to $273.32 instead of $624.13."
-us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC ($6,000) against a $4,081.20 tax, which gave a $1,918.80 ACTC instead of $6,600 − $3,949.20 = $2,650.80. It then inflated 2024 EITC parameters by 5% to a $25,914 phase-out start and got a $1,042 EITC instead of $624.13."
-us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed the earned-income limit capped the ACTC at $3,200, but 15% × ($60,000 − $2,500) = $8,625 does not bind. The binding limit is the CTC left after tax: $6,600 − $3,949.20 = $2,650.80. It also left out the $624.13 EITC."
+us,scenario_028,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It wrongly added non-taxable child support to AGI and used the single standard deduction and brackets instead of head-of-household. It then said the $6,000 CTC fully offsets the $4,574 tax but subtracted only $2,290. The nonrefundable CTC applies up to the full liability, so the tax should be $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It found tax of about $4,000 and $6,600 of CTC, but left $2,900 of tax instead of applying the nonrefundable CTC up to the full liability. The $6,600 CTC eliminates the $3,949.20 tax, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its own reasoning concluded that the nonrefundable CTC cuts the roughly $4,077 tax to $0, but it submitted $2,818, a number its derivation never supports. With the correct $24,150 HoH deduction, the $3,949.20 tax is fully offset by the CTC, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It set aside the $1,700-per-child refundable ACTC first and applied only the remainder as a nonrefundable credit. That order is backwards: the CTC offsets tax liability first, and only the unused part becomes refundable ACTC. Applied correctly, the $6,600 CTC wipes out the roughly $3,950 tax, leaving $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly worked out that the nonrefundable CTC reduces the $4,059 tax to $0. It then replaced that with an unsupported $2,114 citing 'ordering and rounding nuances.' No ordering rule leaves tax remaining when the CTC exceeds the liability."
+us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the pre-TCJA $1,000-per-child CTC ($3,000 total), not the 2026 $2,200-per-child CTC ($6,600). The larger credit fully absorbs the roughly $3,950 tax, so nothing should remain."
+us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA expired, using a $12,150 standard deduction, personal exemptions and a $1,000-per-child CTC. In 2026 there are no personal exemptions, the HoH deduction is $24,150 and the CTC is $2,200 per child, and that CTC eliminates the $3,949.20 tax."
+us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It used $29,200, a married-filing-jointly figure, as the HoH standard deduction instead of $24,150. Then, although its own $6,000 CTC exceeds the tax on $30,810, it reported $1,395 instead of $0. A nonrefundable CTC larger than the tentative tax brings it to zero."
+us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026, using personal exemptions, a roughly $12,000 HoH deduction and a $1,000-per-child CTC. Current law keeps the $24,150 HoH deduction with no exemptions and a $2,200-per-child CTC, which fully offsets the $3,949.20 tax."
+us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied post-TCJA-expiration rules: personal exemptions, a roughly $12,350 deduction and a $1,000-per-child CTC. That left $39 of residual tax. Under 2026 law the $6,600 CTC offsets the full $3,949.20 tax, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It gave no derivation. The $491.50 of residual tax implies a CTC smaller than the liability, consistent with the $1,000-per-child credit. The correct derivation gives taxable income of $35,860 and tax of $3,949.20, fully offset by the $6,600 CTC, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there was no answer to score against the $0 reference."
+us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly found the $24,150 deduction, $35,860 of taxable income and a $6,600 CTC. But it treated $1,800 per child as refundable before applying the nonrefundable credit, so only $1,200 offset the tax. The CTC first offsets liability up to the full $3,949.20, and only the excess becomes ACTC, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It left about $3,000 of tax after the CTC, which means it applied only part of the credit, or a $1,000-per-child credit. The $6,600 CTC exceeds the $3,949.20 tentative tax, so tax before refundable credits is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It counted non-taxable child support in AGI and reported $7,342. That is more than the tentative tax on this income even before credits, so the HoH deduction, brackets and CTC were not applied properly. The correct path is $35,860 of taxable income and $3,949.20 of tax, fully offset by the $6,600 CTC, for $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It computed $4,140.20 of tax and noted that the $6,600 CTC is enough to reduce it to zero. It then submitted $1,956, which is $4,140.20 minus $2,184.20, a figure it had mislabeled as the nonrefundable credit needed. The nonrefundable CTC used equals the full tax, so the result is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave no derivation. The $2,890 of residual tax implies it applied far less than the full CTC against the liability. Tax of $3,949.20 on $35,860 is fully offset by the $6,600 CTC, so the answer is $0."
+us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired, used a 15% bracket and a $1,000-per-child CTC, and even left out the personal exemptions its own expiration premise requires. 2026 law uses the $24,150 HoH deduction and 10%/12% brackets, giving $3,949.20 of tax that the $6,600 CTC fully offsets."
+us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there was no answer to score against the $0 reference."
+us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It used a $2,000-per-child CTC instead of $2,200 and subtracted $5,100 of refundable ACTC first, leaving only $900 as nonrefundable. The nonrefundable CTC is applied first up to the full tax liability, which zeroes out the roughly $3,950 tax."
+us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work found an ACTC of about $2,665 plus an EITC of about $251, but it submitted $5,100, which is just the $1,700 × 3 refundable cap. That cap does not apply here, because the ACTC is limited to the $2,650.80 of CTC left after the $3,949.20 tax. Its EITC also used an $8,046 maximum and a $23,000 phaseout start instead of $8,231 and $23,890, which gives $624.13."
+us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It started the 3-child EITC phaseout at $23,000 instead of the 2026 HoH threshold of $23,890, so its EITC came out at $436.70 instead of $624.13. It also overstated pre-credit tax at $3,955 instead of $3,949.20, which made its ACTC $2,644.80 instead of $2,650.80."
+us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It wrongly put non-taxable child support into income and said no EITC applies. In fact the 3-child EITC at $60,010 AGI is still $624.13. Its $3,850 ACTC is an arbitrary figure: the ACTC is limited to the $6,600 CTC minus the $3,949.20 tax, which is $2,650.80, not the $5,100 cap or a 15%-of-earnings figure."
+us,scenario_028,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It said the tax absorbs the whole CTC, but pre-credit tax is only $3,949.20 against a $6,600 CTC, which leaves $2,650.80 refundable. It also said there is no EITC, but the 3-child EITC does not fully phase out until about $62,975, so $624.13 remains at $60,010."
+us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC ($6,000 total) instead of $2,200 ($6,600), so its unused CTC came out at $2,037 instead of $2,650.80. It also set EITC to zero using an outdated ~$59,899 phaseout endpoint. Under 2026 parameters the credit is $8,231 − 21.06% × ($60,010 − $23,890) = $624.13."
+us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $6,000 CTC, overstated tax at about $4,800 (the actual figure is $3,949.20), and set EITC to zero. Its own work gave an ACTC of about $1,200, yet it submitted an unrelated $4,500. The correct total is the $2,650.80 of unused CTC from $6,600, plus a $624.13 EITC."
+us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,other,False,"It paid the full $1,700 × 3 = $5,100 refundable cap and never limited the ACTC to the CTC left after the nonrefundable part offsets tax, which is $6,600 − $3,949.20 = $2,650.80. It also wrongly dropped the $624.13 EITC as if income were above the 3-child phaseout."
+us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC, a $22,500 standard deduction, and 2025 brackets, so it got $1,830 of unused CTC instead of $6,600 − $3,949.20 = $2,650.80. It also put the end of the 3-child EITC phaseout at about $58,000, when the 2026 endpoint is about $62,975, which leaves a $624.13 EITC."
+us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It assumed EITC ends at about $56,838 and set it to zero, even though the 2026 3-child phaseout leaves $624.13 at $60,010. Its $2,000 ACTC is an unsupported estimate; the actual ACTC is $6,600 CTC minus $3,949.20 tax, which is $2,650.80."
+us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC and a $4,170 tax, so its ACTC was $1,830 instead of the 2026 OBBBA result: $6,600 − $3,949.20 = $2,650.80. Its EITC of $165 is too low; 2026 parameters ($8,231 maximum, $23,890 start, 21.06%) give $624.13."
+us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It said tax liability exceeds the CTC, but pre-credit tax is only $3,949.20 against a $6,600 CTC, which leaves $2,650.80 refundable. It also wrongly treated the EITC as fully phased out at $60,000, when $624.13 remains under the 2026 3-child phaseout."
+us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,other,False,"It claimed the nonrefundable CTC was fully used against tax and so set the refundable CTC to zero. In fact the $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, and that amount is refundable within the $5,100 cap. Its EITC of $662.87 also overshoots the correct $624.13."
+us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC ($6,000) instead of OBBBA's 2026 $2,200 ($6,600) and overstated tax at $4,073.20, so its ACTC was $1,926.80 instead of $2,650.80. Its EITC used an $8,271 maximum and a $24,024 threshold instead of $8,231 and $23,890, which gave $692.35 instead of $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the CTC as having reverted to the post-TCJA amount (about $1,000 per child), so only $66 was left over. Under OBBBA the 2026 CTC is $2,200 per child, which leaves $2,650.80 refundable after the $3,949.20 tax. Its EITC of $524 is also below the correct $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It gave zero refundable credits without computing anything. The $6,600 CTC exceeds the $3,949.20 tax, which leaves a $2,650.80 refundable CTC, and the 3-child EITC still pays $624.13 at $60,010 AGI."
+us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It claimed the CTC was fully used against tax and set the refundable CTC to zero. But the $6,600 CTC minus the $3,949.20 tax leaves $2,650.80, which is refundable within the $1,700-per-child cap. Its EITC of $574 also falls short of $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, as if TCJA had expired) instead of OBBBA's $2,200 per child, and overstated tax at $3,039 (the correct figure is $3,949.20). That left no unused CTC, when the actual unused amount is $2,650.80. It also wrongly said the household is over the EITC limit, but $624.13 of EITC remains at $60,010."
+us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It said phaseouts and tax liability wipe out all refundable credits. But the $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, which is refundable, and the 3-child EITC does not fully phase out until about $62,975, which leaves $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of OBBBA's 2026 $6,600 and overstated tax at $4,089.20, so its ACTC was $1,910.80 instead of $2,650.80. It also left out the $624.13 EITC that still remains in the 3-child phaseout range."
+us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It wrongly said income is above the EITC phaseout endpoint (about $62,975 for three children in 2026) and that the CTC is fully absorbed. The correct figures are a $624.13 EITC and a $2,650.80 refundable CTC ($6,600 − $3,949.20)."
+us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $1,838.80 ACTC reflects a $6,000 pre-OBBBA CTC and an overstated tax. The 2026 $2,200-per-child credit gives $6,600 − $3,949.20 = $2,650.80. Its EITC of $327.51 is also too low; the 2026 parameters give $8,231 − 21.06% × ($60,010 − $23,890) = $624.13."
+us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no usable answer for federal_refundable_credits. The correct figure is the $2,650.80 refundable CTC plus the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,other,False,"It assumed an $1,800 refundable amount per child and paid the full $5,400, ignoring that the ACTC is limited to the CTC left after tax: $6,600 − $3,949.20 = $2,650.80, under a $1,700 cap. It also wrongly zeroed out the EITC, which is $624.13 at $60,010 for three children in 2026."
+us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It treated the whole $2,000-per-child CTC as refundable. In fact the refundable portion is the CTC left after offsetting tax, $6,600 − $3,949.20 = $2,650.80, capped at $1,700 per child. It also left out the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed no refundable credits apply at this income. But the $6,600 CTC exceeds the $3,949.20 tax and leaves $2,650.80 refundable, and the 3-child EITC still pays $624.13 at $60,010 AGI."
+us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It understated pre-credit tax at $2,184.20; the correct figure is $3,949.20, from $35,860 of taxable income after the $24,150 HoH standard deduction. That inflated the unused CTC to $4,415.80 instead of $2,650.80, and it then rounded that to $4,444 without any basis. It also dropped the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It got the $2,650.80 refundable CTC right but wrongly said EITC is unavailable. The 2026 3-child EITC phases out from $23,890 at 21.06% and leaves $624.13 at $60,010, so the total is $3,274.93."
+us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It paid the full $1,700 × 3 = $5,100 refundable cap without limiting the ACTC to the CTC left after the $3,949.20 tax ($2,650.80). It also left out the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its refundable CTC of about $2,651 is correct, but it put the EITC at about $650 instead of computing $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That overstated the total by about $26."
+us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It computed the $2,650.80 refundable CTC correctly but left out the EITC. With three children the credit has not fully phased out at $60,010, and $624.13 remains under 2026 parameters."
+us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It said no refundable credits qualify at this income. The 3-child EITC pays $624.13 at $60,010, and the $6,600 CTC exceeds the $3,949.20 tax, which leaves a $2,650.80 refundable CTC."
+us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It found only about $40 of unused CTC, which means it used a much smaller CTC or a much larger tax than the correct $6,600 CTC and $3,949.20 tax. Those give a $2,650.80 refundable CTC. Its EITC of about $660 also overshoots $624.13."
+us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It found only about $48 of unused CTC, when the 2026 OBBBA $6,600 CTC minus the $3,949.20 tax leaves $2,650.80. Its EITC used an $8,239 maximum and a $23,910 threshold, which gave $636 instead of $624.13."
+us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,other,False,"It made a subtraction error: $6,600 minus its own $3,948 tax is $2,652, not the $2,952 it reported, so its ACTC is overstated by $300. Its EITC of $661 also overshoots the correct $624.13."
+us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the CTC reverts to $1,000 per child ($3,000), which is then fully used against tax. Under OBBBA the 2026 CTC is $2,200 per child, which leaves $2,650.80 refundable after the $3,949.20 tax. Its EITC used an $8,302 maximum and a $24,099 threshold, which gave $739 instead of $624.13."
+us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"Its refundable CTC of $2,649 is essentially correct, but it put the EITC at about $340 instead of $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That understated the total by about $285."
+us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable answer for federal_refundable_credits. The correct figure is the $2,650.80 refundable CTC plus the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $624.13 is exactly right, but it understated pre-credit tax at $3,838.10. The correct tax is $3,949.20: $35,860 of taxable income after the $24,150 HoH deduction, with 10% on the first $17,700 and 12% above. That inflated the ACTC to $2,761.90 instead of $2,650.80."
+us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,other,False,"It took the ACTC as the lesser of the 15% earnings formula and the $5,100 cap, and skipped the limit to the CTC left after tax: $6,600 − $3,949.20 = $2,650.80. It also put the end of the EITC phaseout at about $61,500 and below $60,000 income, when the 2026 3-child endpoint is about $62,975, which leaves $624.13."
+us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Its refundable CTC of $2,650.80 is correct, but it started the EITC phaseout at $22,210 with an $8,234 maximum instead of the 2026 HoH $23,890 and $8,231. That gave $273.32 instead of $624.13."
+us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC ($6,000) and a $4,081.20 tax, so its ACTC was $1,918.80 instead of the OBBBA result: $6,600 − $3,949.20 = $2,650.80. It also inflated older EITC parameters ad hoc to a $8,222 maximum and a $25,914 threshold, which gave $1,042 instead of $624.13."
+us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It said the 15%-of-earnings-over-$2,500 limit binds, but that limit is $8,625 and does not bind. The binding limit is the CTC left after tax, $2,650.80, and its $3,200 figure does not come from either limit. It also left out the $624.13 EITC."
us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got the right parts: $3,720 Social Security, $870 Medicare and $42 PA employee UC (0.07% of $60,000), and it stated the total as $4,632. It then submitted 5,730, a number its own derivation never reaches. The miss is a transcription or consistency error at the output step, not a policy error."
-us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only employee OASDI plus Medicare ($4,590). It left out Pennsylvania's mandatory employee unemployment compensation withholding of 0.07% of gross wages, which adds $42."
-us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It said PA has no employee payroll tax and pointed to PA's corporate taxes. Pennsylvania in fact withholds an employee UC contribution of 0.07% of all gross wages, which is $42 on $60,000, so it undercounted by $42."
-us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said 'PA has no mandatory employee payroll tax counted here' and stopped at FICA of $4,590. It left out Pennsylvania's employee UC withholding of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It summed only employee Social Security and Medicare ($4,590). It never added Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% x $60,000 = $42."
-us,scenario_028,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It counted only the 7.65% FICA ($4,590). It left out Pennsylvania's employee UC withholding of 0.07% of gross wages, which adds $42 for a total of $4,632."
-us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It said Pennsylvania has 'no separate employee payroll tax'. PA does withhold an employee unemployment compensation contribution of 0.07% of total gross wages ($42 here), which the output definition's mandatory state payroll taxes include."
-us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It said Pennsylvania has 'no state disability/UI employee contribution'. PA is one of the states that collects an employee UI contribution, 0.07% of all gross wages, so it missed $42."
-us,scenario_028,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It said no PA employee state payroll tax is modeled and stopped at $4,590. It left out the PA employee UC contribution of 0.07% x $60,000 = $42."
-us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It said there is no PA state employee payroll tax. Pennsylvania withholds an employee unemployment compensation contribution of 0.07% of gross wages, so it undercounted by $42."
-us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare ($4,590). It never added Pennsylvania's mandatory employee UC withholding of 0.07% of $60,000 ($42)."
-us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,It said there is 'no ... state payroll tax'. Pennsylvania's employee unemployment compensation contribution of 0.07% of gross wages is a mandatory employee state payroll tax worth $42 here.
-us,scenario_028,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at FICA of $4,590. It left out Pennsylvania's employee UC withholding (0.07% x $60,000 = $42), which the payroll_tax definition includes."
-us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed only 6.2% Social Security and 1.45% Medicare ($4,590). It left out the Pennsylvania employee unemployment compensation contribution of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only FICA ($4,590). It never considered mandatory employee state payroll taxes, so it missed Pennsylvania's 0.07% employee UC withholding, $42 on $60,000."
-us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It computed employee FICA only ($4,590). It left out Pennsylvania's mandatory employee UC contribution of 0.07% of gross wages, which adds $42."
-us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so nothing could be scored. The correct derivation is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632."
-us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It said 'PA has no mandatory employee state payroll tax'. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all gross wages, so it missed $42."
-us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It said 'no mandatory state payroll tax applies in PA' and stopped at 7.65% FICA. PA's employee UC withholding of 0.07% x $60,000 = $42 applies."
-us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It made an arithmetic error: 6.2% and 1.45% of $60,000 sum to $4,590, but it reported $4,600. It also left out Pennsylvania's employee UC withholding of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It explicitly excluded any 'mandatory PA employee payroll tax' and computed 7.65% x $60,000 = $4,590. Pennsylvania's employee UC contribution of 0.07% of gross wages ($42) is exactly such a tax."
-us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It correctly included a PA employee unemployment tax but set it at $7. That is 0.07% of the $10,000 employer taxable wage base, not of total wages. Pennsylvania's employee UC contribution has no wage cap and applies to all $60,000 of gross wages, which gives $42."
-us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It computed only 7.65% FICA ($4,590). It left out Pennsylvania's mandatory employee UC withholding of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It treated the absence of an explicitly listed state payroll tax as meaning none applies. Pennsylvania's statutory employee UC contribution of 0.07% of gross wages applies to every PA wage earner, adding $42."
-us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% employee FICA ($4,590). It left out Pennsylvania's employee UC withholding of 0.07% x $60,000 = $42."
-us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It said there is 'no PA employee payroll tax'. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all gross wages, so it missed $42."
-us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax. The PA employee UC contribution of 0.07% of gross wages is mandatory and adds $42 to the $4,590 of FICA."
-us,scenario_028,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee state payroll tax. Pennsylvania withholds 0.07% of gross wages as an employee UC contribution ($42 on $60,000)."
-us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It summed only Social Security and Medicare ($4,590). It left out Pennsylvania's employee unemployment compensation withholding of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"It computed employee FICA only ($4,590). It never added Pennsylvania's mandatory employee UC contribution of 0.07% x $60,000 = $42."
-us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so nothing could be scored. The correct derivation is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632."
-us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It said 'no PolicyEngine-modeled employee state payroll tax applies in PA'. PolicyEngine does model Pennsylvania's employee UC contribution at 0.07% of gross wages, which adds $42."
-us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all wages, so it missed $42."
-us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare ($4,590). It left out Pennsylvania's mandatory employee UC withholding of 0.07% of gross wages ($42)."
-us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It said PA has 'no state disability/unemployment deduction from employees'. Pennsylvania does deduct an employee UC contribution of 0.07% of total gross wages, which is $42 on $60,000."
-us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It computed 7.65% FICA only ($4,590). It left out Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% of gross wages ($42)."
+us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got the correct total: FICA of $4,590 plus PA employee UC of 0.07% × $60,000 = $42, for $4,632. It then submitted $5,730, which none of its stated components support. This was an output-consistency error, not a mistake about the rule."
+us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It counted only employee OASDI and Medicare ($4,590) and left out Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% of all wages ($42). The correct total is $4,632."
+us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It said PA has no employee-side payroll tax, pointing to corporate taxes instead. PA does withhold an employee unemployment compensation contribution of 0.07% of all wages, which is $42 here, so the total should have been $4,632 rather than $4,590."
+us,scenario_028,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It computed only federal FICA ($4,590) and left out Pennsylvania's mandatory 0.07% employee unemployment compensation tax on all wages ($42). The total should have been $4,632."
+us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said PA has no mandatory employee payroll tax, but PA withholds a 0.07% employee unemployment compensation contribution on all wages. Missing that $42 understated the total, which is $4,632."
+us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It summed Social Security and Medicare ($4,590) and never applied Pennsylvania's employee UC tax of 0.07% × $60,000 = $42. The correct total is $4,632."
+us,scenario_028,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It counted only FICA at 7.65% and left out Pennsylvania's mandatory employee unemployment compensation contribution (0.07% of uncapped wages, $42). That understated the total by $42."
+us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It said PA has no separate employee payroll tax, but PA imposes a 0.07% employee unemployment compensation tax on all wages. That adds $42 to FICA, for a total of $4,632."
+us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It specifically claimed PA has no employee UI contribution. Pennsylvania is one of the few states that withholds employee UC tax, at 0.07% of all wages ($42 here), so the total is $4,632."
+us,scenario_028,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It claimed no PA employee payroll tax is modeled, but PolicyEngine applies Pennsylvania's 0.07% employee unemployment compensation tax to the head's $60,000 of wages. That adds $42 to the $4,590 of FICA."
+us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It said there is no PA employee payroll tax and so left out Pennsylvania's 0.07% employee UC contribution on all wages ($42). The total should have been $4,632."
+us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare and never added Pennsylvania's mandatory employee unemployment compensation tax of 0.07% × $60,000 = $42."
+us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It said there is no state payroll tax, but Pennsylvania withholds a 0.07% employee UC contribution on all wages. That adds $42, for a total of $4,632."
+us,scenario_028,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at FICA ($4,590) and left out Pennsylvania's employee unemployment compensation tax (0.07% of uncapped wages, $42)."
+us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed only Social Security and Medicare and never applied Pennsylvania's mandatory 0.07% employee UC tax on the $60,000 of wages ($42)."
+us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,It applied only the 7.65% federal FICA rate and left out Pennsylvania's employee unemployment compensation contribution of $42 (0.07% of all wages).
+us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted only FICA ($4,590) and missed Pennsylvania's mandatory employee UC tax of 0.07% × $60,000 = $42, which the requested output explicitly includes as a state payroll tax."
+us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no usable value or reasoning for payroll_tax. The correct answer is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632."
+us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax, but PA withholds a 0.07% employee unemployment compensation contribution on all wages. That adds $42, for a total of $4,632."
+us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no mandatory state payroll tax applies in PA and so left out Pennsylvania's 0.07% employee UC tax ($42 on $60,000)."
+us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It made an arithmetic error: $3,720 + $870 is $4,590, not $4,600. It also left out Pennsylvania's 0.07% employee UC tax ($42), so it missed the correct total of $4,632."
+us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It deliberately excluded any PA employee payroll tax and applied only 7.65% FICA. Pennsylvania's mandatory employee UC contribution of 0.07% × $60,000 = $42 belongs in the total, which is $4,632."
+us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It correctly identified the PA employee unemployment tax but computed it as $7, which matches applying 0.07% to the $10,000 employer taxable wage base. The employee contribution applies to all wages with no cap, so it is 0.07% × $60,000 = $42 and the total is $4,632."
+us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It applied only 7.65% FICA and left out Pennsylvania's mandatory 0.07% employee unemployment compensation tax on all wages ($42).
+us,scenario_028,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It wrote that no mandatory employee state payroll tax is specified, but PA's employee UC tax comes from statute and doesn't need to be listed as a household fact. At 0.07% of $60,000 it adds $42."
+us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,It applied only the 7.65% federal FICA rate and never added Pennsylvania's employee unemployment compensation tax of 0.07% of wages ($42).
+us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It said there is no PA employee payroll tax, but Pennsylvania withholds a 0.07% employee UC contribution on all wages. That adds $42 to FICA, for a total of $4,632."
+us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax and so left out the 0.07% employee unemployment compensation tax ($42 on $60,000)."
+us,scenario_028,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee state payroll tax and so missed PA's 0.07% employee UC contribution on uncapped wages ($42). The total should have been $4,632."
+us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It computed only Social Security and Medicare and never applied Pennsylvania's mandatory employee unemployment compensation tax of 0.07% × $60,000 = $42."
+us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"It stopped at federal FICA ($4,590) and left out Pennsylvania's 0.07% employee UC tax on all wages ($42)."
+us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no usable value or reasoning for payroll_tax. The correct answer is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632."
+us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It said PolicyEngine models no PA employee state payroll tax, but PolicyEngine applies Pennsylvania's 0.07% employee unemployment compensation contribution to all wages. That adds $42 to the $4,590 of FICA."
+us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax and so left out the 0.07% employee UC tax on all wages ($42). The total is $4,632."
+us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and never added Pennsylvania's mandatory 0.07% employee unemployment compensation tax ($42 on $60,000)."
+us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It specifically claimed PA has no employee unemployment deduction beyond income tax withholding. Pennsylvania does withhold employee UC tax at 0.07% of all wages, which adds $42, for a total of $4,632."
+us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied only 7.65% FICA and left out Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% × $60,000 = $42."
us,scenario_028,reduced_price_school_meals_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It counted the $61,277 correctly and first found it above the 185% line. It then switched to an assumed uprated 2026 guideline with 185% at about $61,300. 185% of the $33,000 guideline for four is $61,050, so $61,277 exceeds the reduced-price limit by $227 and the household does not qualify."
us,scenario_028,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used $60,010 of income (wages plus interest) and left out the $1,267 of child support received, which counts as school-meal income. Countable income is $61,277, above the $61,050 threshold it cited itself, so the household is over 185% of the guideline and not eligible."
us,scenario_028,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It counted $61,277 correctly but set 185% of the guideline at about $61,975, which implies a guideline of roughly $33,500. The correct limit is 185% of $33,000, or $61,050, and $61,277 exceeds it (ratio 1.86), so the household is not eligible for reduced-price meals."
@@ -2541,131 +2623,135 @@ us,scenario_028,reduced_price_school_meals_eligible,ox-alpha,llm_error,taxable_i
us,scenario_028,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a Pennsylvania standard deduction and then, contradicting itself, inflated the base to ""approximately $69,000"" ($2,118 / 0.0307 = $69,000) — a figure with no derivation that also swept in the $1,267 of child support, which is not one of Pennsylvania's eight taxable classes of income. Pennsylvania taxes compensation of $60,000 plus interest of $10 with no standard deduction and no exemptions, so the base is $60,010 and the tax is 3.07% × $60,010 = $1,842.31."
-us,scenario_028,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted four personal exemptions of $4,400 each ($17,600) from the correct $60,010 PA base, importing the repealed federal personal-exemption concept into a state that has none. Pennsylvania's personal income tax allows no standard deduction and no personal exemptions; its only income-based relief is the Schedule SP tax forgiveness credit, which this household's eligibility income far exceeds, so the tax is the full 3.07% × $60,010 = $1,842.31."
-us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model identified the correct rule and base — Pennsylvania's flat 3.07% on $60,010 — but then reported $1,845 instead of executing the multiplication, which yields $1,842.31. The $2.69 gap is a pure arithmetic slip in the final product, not a policy error."
-us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,missing_output,False,"The model's explanation derived Pennsylvania's flat 3.07% on the $60,010 base of compensation plus interest and stated $1,842.31, exactly matching the reference, but the submitted numeric `value` field for state_income_tax_before_refundable_credits was 0. The failure is in populating the structured output field, not in the tax reasoning."
-us,scenario_028,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so the key is absent from the submitted outputs. The correct derivation is Pennsylvania's flat 3.07% applied to $60,010 of compensation and interest, giving $1,842.31."
-us,scenario_028,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model computed the correct $1,842 base tax, then applied Schedule SP tax forgiveness using a fabricated $32,000 eligibility-income base for an unmarried claimant; the actual PA Schedule SP limit is $6,500 for the claimant plus $9,500 per dependent, or $35,000 here. The household's eligibility income of $61,277 (including the $1,267 of child support, which counts toward SP eligibility income even though it is untaxed) exceeds that limit by a wide margin, so the forgiveness percentage is zero and the invented $15 reduction is spurious."
-us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule — flat 3.07% on wages plus interest with no PA deductions — and then reported ""roughly $1,650"" without performing the multiplication; 3.07% × $60,010 = $1,842.31, and $1,650 corresponds to an effective rate of 2.75% that no PA provision produces. The error is an unexecuted computation substituted with a guess."
-us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted a Pennsylvania ""standard deduction and nonrefundable credits"" and returned $3,480, which implies an effective rate of 5.8% on $60,010 — nearly double Pennsylvania's statutory flat rate. Pennsylvania levies a single 3.07% rate on the eight classes of income with no standard deduction, giving $1,842.31."
-us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model equated PA taxable income with federal AGI and then subtracted a $17,800 head-of-household standard deduction, a purely federal construct; Pennsylvania has no standard deduction and no filing-status-based allowances. Its base is the sum of the PA income classes — $60,000 compensation plus $10 interest — taxed in full at 3.07% for $1,842.31."
-us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model correctly derived $1,842.31 and then subtracted $1.55 for the PA Child and Dependent Care Enhancement Tax Credit, which is a refundable credit reported under state_refundable_credits and therefore excluded from this before-refundable-credits output. The credit is also zero for this household: it is a percentage of the federal CDCC, and no childcare expenses are listed, so the federal credit base is $0."
-us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit such as a state EITC and looked only at nonrefundable Tax Forgiveness. It missed PA's refundable EITC, which is 10% of the federal EITC. The household's $624.13 federal EITC (three children, $60,000 earnings) gives a PA credit of $62.41."
-us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only the PA child and dependent care credit and dropped it because there are no child care expenses. It missed PA's refundable EITC, which is 10% of the household's $624.13 federal EITC, or $62.41."
-us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model knew PA has an earned income tax credit but wrongly called it nonrefundable. PA's EITC is refundable and equals 10% of the federal EITC. With a $624.13 federal EITC, this household gets $62.41."
-us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated Tax Forgiveness as PA's only relevant credit and said no other refundable credit applies. It missed the refundable PA EITC at 10% of the federal EITC. The household still has a $624.13 federal EITC in the phase-out range, so the PA credit is $62.41."
-us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model called Tax Forgiveness PA's main refundable mechanism and said no other refundable credit exists. It missed the refundable PA EITC, which is 10% of the federal EITC ($624.13 × 10% = $62.41)."
-us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies and missed PA's refundable EITC at 10% of the federal EITC. This head of household with three children has a $624.13 federal EITC, which yields $62.41."
-us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated outright that Pennsylvania has no state EITC. That is wrong: PA now has a refundable EITC equal to 10% of the federal EITC. Here that is 10% of $624.13, or $62.41."
-us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit comparable to a state EITC and looked only at Tax Forgiveness. It missed PA's refundable EITC at 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It never applied PA's refundable EITC at 10% of the federal EITC, which is $62.41 given the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model said PA offers no refundable credit at this income level. It missed that PA's refundable EITC is 10% of the federal EITC, and the federal EITC is still $624.13 at $60,000 of earnings with three children. That makes the PA credit $62.41."
-us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness income limits, and it also miscounted the household as four people. It never applied PA's refundable EITC at 10% of the $624.13 federal EITC, which is $62.41."
-us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said PA provides no refundable credit here. It missed the refundable PA EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said PA has no refundable individual income tax credits at all. That ignores PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this household."
-us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no PA refundable credit applies at this income level. It missed the refundable PA EITC at 10% of the federal EITC. The household's federal EITC is $624.13 near the end of the phase-out, so the PA credit is $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 from a $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model looked only at the PA child and dependent care enhancement credit and found no child care expenses. It missed the refundable PA EITC at 10% of the federal EITC, which is 10% of $624.13, or $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It missed the PA EITC, a refundable credit equal to 10% of the federal EITC, which yields $62.41 here."
-us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said no state refundable credit is available. It missed PA's refundable EITC at 10% of the federal EITC, and the household's $624.13 federal EITC gives $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this three-child household with a $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It never computed PA's refundable EITC at 10% of the federal EITC, which is $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model said the household is ineligible for any PA refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_refundable_credits, so there was nothing to score. The correct value is 10% of the household's $624.13 federal EITC, or $62.41, from PA's refundable EITC."
-us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model looked only at the Property Tax/Rent Rebate, which requires age 65 or a disability, and concluded PA has no refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here."
-us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said the household facts support no PA refundable credit. The facts do support one: three qualifying children and $60,000 of earnings give a $624.13 federal EITC. PA's refundable EITC is 10% of that, or $62.41."
-us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed the income was too high for any major refundable credit. At $60,000 of earnings with three children, the household is still in the federal EITC phase-out with a $624.13 credit. PA's refundable EITC pays 10% of that, or $62.41."
-us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for child care expenses or other special inputs and found none. It missed that PA's refundable EITC depends only on the federal EITC, which this household receives ($624.13). That makes the PA credit 10% of it, or $62.41."
-us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It never applied PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this household."
-us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model said no PA refundable credit applies at this income. It missed that the household still gets a $624.13 federal EITC, and PA's refundable EITC pays 10% of it, or $62.41."
-us,scenario_028,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model used the right rule, PA's refundable EITC at 10% of the federal EITC, but guessed the federal EITC at about $650. The correct federal EITC is $624.13, from the 2026 three-child phase-out at $60,010 of AGI. Its $65 answer is 10% of that overestimate, not the correct $62.41."
-us,scenario_028,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said there are no state refundable credits. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here."
-us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said no refundable PA personal income tax credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 from the $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model said Pennsylvania has no refundable individual income tax credit that applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41."
-us,scenario_028,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model stated outright that PA has no refundable earned income credit and looked only at Tax Forgiveness. PA's refundable EITC pays 10% of the federal EITC, which is $62.41 for this household."
-us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model stated outright that PA has no state EITC. PA's refundable EITC is 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness and dropped it as nonrefundable and income-ineligible. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here."
-us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness income limits, and it also miscounted the household as four people. It missed PA's refundable EITC at 10% of the household's $624.13 federal EITC, which is $62.41."
-us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model stated outright that PA has no state EITC or any refundable credit. PA's refundable EITC is 10% of the federal EITC, which is $62.41 for this household."
-us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model said none of PA's limited refundable credits apply. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness eligibility at $60,010. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here."
-us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model stated outright that PA has no refundable state EITC. PA's refundable EITC is 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this three-child household."
+us,scenario_028,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted the $1,267 of child support as taxable, but child support is not in any PA taxable income class. It then made up a PA taxable income of about $69,000, which is more than its own stated gross income. Its $2,118 equals 3.07% × about $68,990, not 3.07% × $60,010 = $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model's explanation correctly applies the flat 3.07% to $60,010 and ends with 'value = 1842', but the numeric value it submitted was 0. The miss comes from the mismatch between the submitted value and the explanation, not from the tax calculation."
+us,scenario_028,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted four made-up $4,400 personal exemptions ($17,600) before applying the 3.07% rate. PA allows no personal or dependent exemptions, so the full $60,010 is taxed, giving $1,842.31 instead of $1,302."
+us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the right method (3.07% of $60,010) but made an arithmetic or rounding error. It reported $1,845 when the product is $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model's explanation derives exactly $1,842.31: it applies no standard deduction or exemptions, excludes child support, and multiplies $60,010 by 3.07%. It ends with 'value = 1842.31', but the numeric value it submitted was 0. The miss is a contract mismatch, not a calculation error."
+us,scenario_028,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for this output. The correct value is 3.07% × $60,010 = $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model applied PA Schedule SP tax forgiveness using a made-up $32,000 base. The actual full-forgiveness limit for an unmarried claimant with three dependents is $6,500 + 3 × $9,500 = $35,000, and forgiveness drops 10% for each $250 above that. The model also left the $1,267 of child support out of eligibility income. With eligibility income of about $61,277, the household gets no forgiveness, so subtracting $15 was wrong; the tax is the full $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model correctly named the flat 3.07% rate on wages and interest but did not do the multiplication. Its $1,650 implies a taxable base of about $53,700, when the full $60,010 is taxable with no deductions, giving $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model treated PA tax as a federal-style calculation with a standard deduction and graduated rates. PA has no standard deduction and taxes all income at a flat 3.07%. Its $3,480 implies about $113,000 of income taxed at 3.07%, or an effective rate of about 5.8% on $60,010, either of which is wrong. The correct tax is $1,842.31."
+us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model subtracted a made-up $17,800 PA head-of-household standard deduction. PA has no standard deduction, so 3.07% applies to the full $60,010, giving $1,842.31 instead of $1,296."
+us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model correctly computed $1,842.31 and then subtracted a made-up $1.55 PA Child and Dependent Care credit. The household has no dependent care expenses, so no such credit exists. Because PA's dependent care credit is refundable, it would not reduce tax before refundable credits in any case."
+us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said PA has no state EITC and only the nonrefundable Tax Forgiveness program. It missed the refundable Working Pennsylvanians Tax Credit, which pays 10% of the federal EITC. With the $624.13 federal EITC, that credit is $62.41."
+us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only the PA child and dependent care credit and stopped at $0 because there were no care expenses. It missed PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed PA's EIC is nonrefundable. PA's Working Pennsylvanians Tax Credit is a refundable credit equal to 10% of the federal EITC, which gives $62.41 for this household's $624.13 federal EITC."
+us,scenario_028,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model concluded that no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC. This three-child household receives $624.13 of federal EITC, so the PA credit is $62.41."
+us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated Tax Forgiveness as the only possible PA refundable credit and ruled it out on income. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 here."
+us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model limited its analysis to Tax Forgiveness and asserted PA has no other refundable credits. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It overlooked the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC. With the $624.13 federal EITC, the credit is $62.41."
+us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated that Pennsylvania has no state EITC. The Working Pennsylvanians Tax Credit is a refundable state EITC set at 10% of the federal EITC, so this household with a $624.13 federal EITC gets $62.41."
+us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit comparable to a state EITC and looked only at Tax Forgiveness. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 on this household's $624.13 federal EITC."
+us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed no PA refundable credit is available at this income level. The Working Pennsylvanians Tax Credit is 10% of any positive federal EITC, and the household's $624.13 federal EITC yields $62.41."
+us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model tested only Tax Forgiveness income thresholds and also miscounted the family as four people. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said PA provides no refundable credit for this household. It missed the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC, which comes to $62.41 here."
+us,scenario_028,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed PA offers no refundable individual income tax credits. That is outdated: the Working Pennsylvanians Tax Credit is a refundable 10%-of-federal-EITC credit, which equals $62.41 for this household."
+us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model found no applicable PA refundable credits for 2026. It missed the Working Pennsylvanians Tax Credit, which equals 10% of the federal EITC; applied to the $624.13 federal EITC, that is $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credits apply. It omitted PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model checked only the PA Child and Dependent Care Enhancement credit, which requires care expenses. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model concluded no PA refundable credit is available. It missed the Working Pennsylvanians Tax Credit, which pays 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said no state refundable credits apply. It ignored PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC and equals $62.41 here."
+us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model said the household does not qualify for PA refundable credits. Any household with a positive federal EITC qualifies for the Working Pennsylvanians Tax Credit at 10% of that amount, so this household gets $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the household does not qualify for refundable PA credits. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model found the household ineligible for any PA refundable credit. It overlooked the Working Pennsylvanians Tax Credit, a refundable credit worth 10% of the federal EITC, which is $62.41 here."
+us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was no answer to score. The correct value is 10% of the $624.13 federal EITC under the Working Pennsylvanians Tax Credit, or $62.41."
+us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model treated the Property Tax/Rent Rebate, which requires age 65 or disability, as the only candidate. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no PA refundable credit indicated by the household facts. It missed the Working Pennsylvanians Tax Credit, which follows directly from the household's $624.13 federal EITC and comes to $62.41 at 10%."
+us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed income was too high for any refundable credit. The household still receives a $624.13 federal EITC, and PA's refundable Working Pennsylvanians Tax Credit pays 10% of it, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for child care expenses and other special inputs, found none, and returned $0. It missed the Working Pennsylvanians Tax Credit, which needs only a federal EITC and equals 10% of the $624.13 federal credit, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It omitted the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC, which is $62.41 here."
+us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model assumed no PA refundable credit applies at this income. The household still has a $624.13 federal EITC, and the Working Pennsylvanians Tax Credit pays 10% of it, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model correctly applied PA's 10%-of-federal-EITC refundable credit but rounded the federal EITC to about $650. The exact 2026 three-child phase-out leaves $624.13 at roughly $60,010 of income, so the PA credit is $62.41, not $65."
+us,scenario_028,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit is indicated. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model asserted there are no state refundable credits. It missed PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said no PA refundable credits apply. It overlooked the Working Pennsylvanians Tax Credit, which pays 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Pennsylvania has no applicable refundable credits. The Working Pennsylvanians Tax Credit is a refundable 10%-of-federal-EITC credit, which is $62.41 for this household."
+us,scenario_028,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model asserted PA has no refundable earned income credit and looked only at Tax Forgiveness. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model stated that PA has no state EITC. The Working Pennsylvanians Tax Credit is a refundable state EITC at 10% of the federal credit, so this household gets $62.41."
+us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model considered only Tax Forgiveness and rejected it on income. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model tested only Tax Forgiveness thresholds and also miscounted the family as four people. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model claimed PA has no state EITC. The Working Pennsylvanians Tax Credit is a refundable credit equal to 10% of the federal EITC, which gives $62.41 on this household's $624.13 federal EITC."
+us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model said none of PA's refundable credits apply. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model checked only Tax Forgiveness eligibility. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model stated that PA has no refundable state EITC. The Working Pennsylvanians Tax Credit is a refundable credit at 10% of the federal EITC, worth $62.41 for this household."
+us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said no PA refundable credits apply. It missed the Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 on the $624.13 federal credit."
us,scenario_028,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It stated ""AGI is $50,312 ($50,000 taxable financial assistance + $312 taxable interest),"" treating the non-taxable financial-assistance input as ordinary taxable income when that variable enters no AGI component, leaving actual AGI at $312 of taxable interest alone. It compounded this by applying a pre-TCJA 2026 schedule — an $8,600 single standard deduction with 10%/15% brackets — instead of the 2026 single standard deduction of about $16,100 under the permanent OBBBA rates. Either correction alone puts the $312 of interest below the standard deduction, yielding $0 rather than $5,676.80."
us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It explicitly ""treated [the $50,000 financial assistance] as taxable income for federal tax purposes"" and added it to the $312 of interest, but financial assistance is a non-taxable receipt that contributes nothing to AGI, so the return reports only $312 of taxable interest. Its $5,236 is consistent with running a bracket schedule on roughly $45,700 of taxable income after subtracting a standard deduction from a $50,312 base. The correct derivation is $312 AGI minus the 2026 single standard deduction of about $16,100, giving $0 taxable income and $0 tax before refundable credits."
us,scenario_029,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It recharacterized the $50,000 of non-taxable financial assistance as ""wages"" to reach AGI of $50,312, when the only AGI item is $312 of taxable interest, which falls below the roughly $16,100 single standard deduction it itself cited (as $16,200) and produces $0 taxable income. It then applied a $2,000 child tax credit to a childless 21-year-old single filer, who has no qualifying child under 17 and so gets no CTC at all. Both errors point the same direction; the correct answer is $0, not $2,051."
-us,scenario_029,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added the $50,000 of financial assistance to the $312 of taxable interest and treated the $50,312 total as MAGI; financial assistance is a non-taxable receipt that is excluded from Medicaid MAGI, so countable income is the $312 of interest alone, or 0.02 x FPL. It compounded the error by invoking a resource test on the $54,985 of bank and stock assets, which 42 CFR 435.603(g) forbids for MAGI-based groups such as Oklahoma's ACA adult expansion category."
-us,scenario_029,head_medicaid_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that Oklahoma never adopted the ACA expansion; Oklahoma adopted adult expansion by State Question 802 effective July 1, 2021, covering childless adults 19-64 to 138% FPL, which is exactly the ADULT category the engine assigns to this 21-year-old at 0.02 x FPL. It then applied an asset limit to the $54,985 in bank and stock holdings and treated employer-sponsored insurance as disqualifying, but MAGI-based categories carry no resource test and ESI bars the ACA premium tax credit rather than Medicaid."
-us,scenario_029,head_medicaid_eligible,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It counted the $50,000 financial assistance as Medicaid income, producing ~$50,312 against the 138% FPL adult expansion limit of roughly $21,600. Financial assistance is excluded from MAGI, so the countable figure is the $312 of taxable interest, 0.02 x FPL, which clears the adult expansion income test with enormous margin."
-us,scenario_029,head_medicaid_eligible,claude-opus-4.8,llm_error,other,False,"Its reasoning reached the correct derivation in full — the $50,000 financial assistance is not MAGI income, MAGI groups apply no asset test, Oklahoma expanded, and $312 of interest sits below 138% FPL — and it concluded verbatim that 'they are eligible.' It then submitted value = 0, contradicting its own stated conclusion; the substantive analysis was right and the emitted binary was inverted."
-us,scenario_029,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the $50,000 financial assistance as income against Oklahoma's 138% FPL adult expansion limit. That receipt is excluded from Medicaid MAGI, leaving only the $312 of taxable interest (0.02 x FPL), so the adult expansion income test is met."
-us,scenario_029,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It summed the $50,000 financial assistance with the $312 of taxable interest into ~$50,312 of countable MAGI; financial assistance is not MAGI income, so the correct figure is $312, or 0.02 x FPL. It also applied a bare 133% FPL threshold rather than the effective 138% (133% plus the 5-percentage-point income disregard), and treated employer-sponsored insurance as bearing on Medicaid eligibility when ESI only affects premium tax credit eligibility."
-us,scenario_029,head_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It explicitly counted the $50,000 financial assistance as income, putting the head above the ~$21,000 expansion limit; that receipt is outside MAGI, so countable income is $312 (0.02 x FPL). Its fallback claim that the head is not otherwise categorically eligible ignores that the ACA adult expansion group is itself the categorical pathway for a non-pregnant, non-disabled, childless 21-year-old under 65."
-us,scenario_029,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It stated outright that the $50,000 financial assistance counts as income above 138% FPL. Financial assistance is excluded from Medicaid MAGI, leaving $312 of taxable interest at 0.02 x FPL, which satisfies Oklahoma's adult expansion income test."
-us,scenario_029,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed household MAGI as $50,312 by including the $50,000 financial assistance alongside the taxable interest. Under MAGI rules that transfer is not countable, so the comparison against the ~$21,597 threshold should have used $312, placing the head at 0.02 x FPL and squarely inside the ADULT expansion group."
-us,scenario_029,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted that income exceeds Oklahoma's single-adult Medicaid threshold with no derivation; the only figure in the household that could exceed 138% FPL for a one-person unit is the $50,000 financial assistance, which is excluded from MAGI. Counting only the $312 of taxable interest gives 0.02 x FPL, well inside the adult expansion limit."
-us,scenario_029,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave a bare denial with no derivation. The correct computation counts only the $312 of taxable interest as MAGI — 0.02 x FPL — placing the 21-year-old in Oklahoma's ACA adult expansion group; its zero is consistent with treating the $50,000 financial assistance as countable income, the only household figure capable of exceeding 138% FPL for a one-person unit."
-us,scenario_029,head_medicaid_eligible,glm-5.2,llm_error,taxable_income_or_deductions,False,"It totaled the $50,000 financial assistance and $312 interest into $50,312 against a ~$22,000 expansion limit; the financial assistance is not MAGI income, so the test uses $312, or 0.02 x FPL. Its supporting point that the head has employer-sponsored insurance is irrelevant, since ESI disqualifies from the premium tax credit, not from Medicaid."
-us,scenario_029,head_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"It claimed income above Oklahoma's MAGI limit, which only holds if the $50,000 financial assistance is counted; excluding that non-MAGI receipt leaves $312 of taxable interest at 0.02 x FPL. Its second ground, availability of employer-sponsored insurance, does not affect Medicaid eligibility at all."
-us,scenario_029,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied pre-ACA categorical rules, demanding disability, pregnancy, or a dependent child before Medicaid can attach. Oklahoma's ACA adult expansion group requires only age 19-64, no Medicare, satisfactory immigration status, and MAGI at or below 138% FPL — all met here at age 21 with MAGI of $312, or 0.02 x FPL."
-us,scenario_029,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It correctly identified the adult expansion MAGI test but fed it the $50,000 financial assistance plus $312 interest. Financial assistance is not included in MAGI, so the one-person unit is tested at $312, 0.02 x FPL, and passes."
-us,scenario_029,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It treated the $50,000 financial assistance as household income exceeding Oklahoma's adult expansion limit. That receipt falls outside MAGI, leaving $312 of taxable interest at 0.02 x FPL, which satisfies the 138% FPL test for the ADULT category."
-us,scenario_029,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It denied eligibility on income, assets, and employer coverage at once, and all three grounds fail: the $50,000 financial assistance is excluded from MAGI so countable income is $312 (0.02 x FPL), MAGI-based categories apply no resource test to the $54,985 in bank and stock assets, and employer-sponsored insurance bars the premium tax credit rather than Medicaid."
-us,scenario_029,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It got Oklahoma's expansion rule right but computed countable income as $50,000 financial assistance plus $312 interest. Financial assistance is not MAGI income, so the head is tested at $312 against roughly $21,000 — 0.02 x FPL — and qualifies under the ADULT category."
-us,scenario_029,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It required a qualifying disability, pregnancy, or other special basis and treated the absence of earnings as removing any eligibility hook. Oklahoma's ACA adult expansion covers a childless, non-disabled 21-year-old purely on MAGI at or below 138% FPL, and this head's MAGI of $312 is 0.02 x FPL, so low income is the qualifying condition rather than a bar."
+us,scenario_029,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added the $50,000 of financial assistance to MAGI, getting about $50,312. That assistance is non-taxable and outside AGI and MAGI, so MAGI is only $312, about 0.02x FPL and far below the 138% expansion limit. Its fallback point about resources also fails, because the ACA adult group has no asset test."
+us,scenario_029,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said Oklahoma did not expand Medicaid, but Oklahoma has covered ACA expansion adults since 2021. A 21-year-old childless adult therefore qualifies with MAGI of $312. It also wrongly applied an asset limit to $54,985 of bank and stock assets and treated ESI as disqualifying, though MAGI categories have no asset test and ESI does not block eligibility."
+us,scenario_029,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Called Oklahoma a non-expansion state with no coverage for childless adults. Oklahoma covers expansion adults aged 19-64 up to 138% FPL, and the head's MAGI of $312 (0.02x FPL) qualifies them for the ADULT category."
+us,scenario_029,head_medicaid_eligible,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance as MAGI income. It is non-taxable support and excluded from MAGI. The head's MAGI is the $312 of interest, about 0.02x FPL, well under the roughly $21,600 limit (138% FPL) for one person."
+us,scenario_029,head_medicaid_eligible,claude-opus-4.8,llm_error,other,False,"Its reasoning got it right: financial assistance is excluded from MAGI, assets are not counted, and $312 of MAGI is below 138% FPL in expansion-state Oklahoma, which means eligible. It then submitted 0, which contradicts its own stated conclusion."
+us,scenario_029,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Treated the $50,000 of financial assistance as income for the expansion test. That assistance is not part of MAGI, so MAGI is only $312 of interest (0.02x FPL), well below the 138% FPL limit."
+us,scenario_029,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance as MAGI income, getting about $50,312. The assistance is non-taxable and excluded, so MAGI is $312, about 0.02x FPL. It also treated ESI as a factor against eligibility, but ESI does not block expansion-adult Medicaid."
+us,scenario_029,head_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Correctly noted that Oklahoma expanded Medicaid, but counted the $50,000 of financial assistance as income. MAGI excludes that non-taxable support, so countable income is $312 (0.02x FPL). The head qualifies in the ACA adult group, which needs no pregnancy, disability or parent status."
+us,scenario_029,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance as income above 138% FPL. Financial assistance is excluded from MAGI, so the head's MAGI is $312 of interest (0.02x FPL), which qualifies for Oklahoma's expansion adult group."
+us,scenario_029,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Computed household MAGI as $50,312 by including the $50,000 of financial assistance. That transfer is not part of AGI or MAGI. MAGI is $312, about 0.02x FPL, well under the roughly $21,597 limit it cited."
+us,scenario_029,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Said income was above the single-adult threshold, which only holds if the $50,000 of financial assistance is counted. The only MAGI income is $312 of taxable interest (0.02x FPL), so the head meets the 138% FPL expansion-adult test."
+us,scenario_029,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no substantive reasoning. The correct derivation is MAGI of $312 interest (0.02x FPL), with the $50,000 of financial assistance excluded, plus age 21 in expansion-state Oklahoma, which makes the head eligible as an ADULT. Its 0 is consistent with counting the $50,000 assistance as income or ignoring the expansion group."
+us,scenario_029,head_medicaid_eligible,glm-5.2,llm_error,taxable_income_or_deductions,False,"Added the $50,000 of financial assistance to the $312 of interest, getting $50,312 of income. Financial assistance is excluded from MAGI, so MAGI is $312, far below 138% FPL. It also wrongly treated employer-sponsored insurance as relevant, though ESI does not affect Medicaid eligibility."
+us,scenario_029,head_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"Said income exceeded Oklahoma's MAGI limits, which requires counting the $50,000 of financial assistance. That assistance is excluded from MAGI, which is $312 (0.02x FPL). Having ESI available does not disqualify an adult from Medicaid."
+us,scenario_029,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Required disability, pregnancy or a dependent child for eligibility and never applied the ACA adult expansion group. In Oklahoma that group covers any non-elderly adult aged 19-64 with MAGI at or below 138% FPL. The head's $312 of MAGI (0.02x FPL) qualifies."
+us,scenario_029,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Correctly applied the expansion-adult MAGI test but put the $50,000 of financial assistance into MAGI. That assistance is non-taxable and excluded. MAGI is $312 (0.02x FPL), well under the limit."
+us,scenario_029,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Treated the $50,000 of financial assistance as household income for the expansion test. MAGI excludes that assistance, leaving only $312 of interest (0.02x FPL), which meets Oklahoma's 138% FPL adult limit."
+us,scenario_029,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"Said income and assets exceeded Medicaid limits and cited ESI. The expansion adult group has no asset test and ignores ESI, and its MAGI excludes the $50,000 of financial assistance. That leaves $312 of MAGI (0.02x FPL), so the head is eligible."
+us,scenario_029,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Correctly noted Oklahoma's 138% FPL expansion but counted the $50,000 of financial assistance, getting $50,312 of countable income. Financial assistance is excluded from MAGI. Countable MAGI is $312 (0.02x FPL), which meets the limit."
+us,scenario_029,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"Looked for a disability, pregnancy or similar categorical basis and found none, missing the ACA adult expansion group. That group requires only age 19-64 and MAGI at or below 138% FPL. The head is 21 with $312 of MAGI (0.02x FPL)."
us,scenario_029,payroll_tax,glm-5.3,llm_error,payroll_tax_base,False,"glm-5.3 recharacterized the $50,000 ""financial assistance"" line as wages and applied the combined 7.65% employee FICA rate (6.2% OASDI + 1.45% HI) to it, yielding $3,825. Financial assistance is a non-employment transfer, not remuneration for services, so it falls outside the FICA wage base of IRC §3121(a) and carries no employee Social Security or Medicare tax; the household's only other income, $312 of taxable interest, is unearned and likewise never in the payroll tax base. Restricting the tax to actual wage and salary income leaves a base of $0 and an employee-side payroll tax of $0."
us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model skipped the SNAP resource test and calculated an allotment solely from monthly interest income and the standard deduction. The household’s $54,985 of bank and stock assets exceeds Oklahoma’s applicable SNAP resource limit, making it ineligible before any maximum-allotment calculation."
us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model built Oklahoma taxable income of $50,312 by adding the $50,000 of financial assistance to the $312 of taxable interest, but financial assistance is a non-taxable transfer that is excluded from federal AGI, which is Oklahoma's starting point, so Oklahoma AGI is $312. It compounded this with an $8,600 standard deduction in place of Oklahoma's single standard deduction plus the $1,000 personal exemption, then ran the graduated 0.25%–4.75% schedule over $41,712 of income that does not exist; with only $312 of interest, Oklahoma taxable income is zero and the tax is $0.00."
us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no arithmetic, but $1,827 is what Oklahoma's graduated schedule yields on roughly $43,000 of taxable income — $50,312 of gross income less about $7,350 of standard deduction and personal exemption — which is reachable only by treating the $50,000 of financial assistance as taxable income. Financial assistance is non-taxable and never enters federal AGI or Oklahoma AGI; the sole taxable item is $312 of interest, which falls below Oklahoma's standard deduction and produces $0.00 of tax."
us,scenario_029,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model's $36,662 base is exactly $50,312 minus a $13,650 deduction, showing it counted the non-taxable $50,000 financial assistance as Oklahoma taxable income and then subtracted a federal-style standard deduction rather than Oklahoma's single standard deduction plus $1,000 personal exemption. Excluding the non-taxable assistance leaves $312 of taxable interest, below every Oklahoma deduction and filing threshold, so the correct liability is $0.00 instead of $1,160."
-us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Correctly named the $40-per-exemption Sales Tax Relief Credit but added the $50,000 of financial assistance to gross household income and disqualified the head against the $20,000 limit. The credit's gross income is only the $312 of interest, which is far under the limit, so one exemption yields $40."
-us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Oklahoma has no refundable individual income tax credits, overlooking the refundable Sales Tax Relief Credit of $40 per exemption for households under the $20,000 gross-income limit. The head's $312 gross income qualifies for $40."
-us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Named the Sales Tax Relief Credit but counted the $50,000 of financial assistance (and pointed to the stock and bank assets) toward the $20,000 income limit. The credit's gross income is only the $312 of interest and the credit has no asset test, so the head receives $40 for one exemption."
-us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Denied the Sales Tax Relief Credit on the grounds of no earned income and 'substantial assets/financial assistance'. The credit has no earned-income or asset test, and gross income of $312 is under the $20,000 limit, so it pays $40 for the single exemption."
-us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said the sales tax relief thresholds were not met and treated take-up as zero, but the prompt assumes filing and take-up. With $312 of gross income under the $20,000 limit, the Sales Tax Relief Credit is $40 for one exemption."
-us,scenario_029,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Treated earned income or dependents as a prerequisite for any Oklahoma refundable credit and never applied the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Considered only the Oklahoma EITC and dismissed every other refundable credit, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with gross income under $20,000; here gross income is $312."
-us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Found no general refundable credit and treated the elderly/disability pathway as the only non-EITC basis, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income under $20,000, which covers this head's $312."
-us,scenario_029,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Said no refundable credits apply 'at this income level' when a very low income is exactly what qualifies for the Sales Tax Relief Credit. Gross income of $312 is under the $20,000 limit, giving $40 for one exemption."
-us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Tied refundable credits to having tax liability, but the Sales Tax Relief Credit is refundable regardless of tax. It pays $40 per exemption when gross income ($312) is under $20,000."
-us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Correctly zeroed the 5% Oklahoma EITC but wrongly concluded no other refundable credit applies, omitting the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Evaluated only the EITC-based credit and missed the refundable Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under $20,000."
-us,scenario_029,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,Reasoned only from the $0 federal EITC and never applied the Oklahoma Sales Tax Relief Credit. That credit gives $40 per exemption at gross income of $312.
-us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Computed gross income as $50,312 by adding the $50,000 of financial assistance and compared it with the $20,000 Sales Tax Relief Credit limit. The credit's gross income is only the $312 of interest, which qualifies the single exemption for $40."
-us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no state refundable credits apply, missing the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in total income for the Sales Tax Relief Credit and so exceeded the $20,000 limit. The credit's gross income is $312 of interest, so the head receives $40 for one exemption."
-us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Found no qualifying refundable credit, overlooking the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to an adult filer with gross income ($312) under the $20,000 limit."
-us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned $0 without applying the Oklahoma Sales Tax Relief Credit. That credit pays $40 for the head's single exemption because gross income of $312 is under $20,000."
-us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"Treated age 65+, disability, or dependents as a prerequisite for the Sales Tax Relief Credit. Those factors only raise the income limit to $50,000; an adult single filer qualifies under the base $20,000 limit, and with $312 of gross income receives $40."
-us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Concluded no Oklahoma refundable credit applies, missing the Sales Tax Relief Credit. That credit gives $40 per exemption when gross income is $312, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,Checked only the Oklahoma EITC (zero because the federal EITC is zero) and never applied the Sales Tax Relief Credit. That credit pays $40 for one exemption at gross income of $312.
-us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Asserted that Oklahoma offers no refundable credit for this household, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Reported no refundable Oklahoma credits, missing the Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under $20,000."
-us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no factual basis for a refundable Oklahoma credit, but the $312 of gross income is itself the basis for the Sales Tax Relief Credit. That income is under the $20,000 limit, giving $40 for one exemption."
-us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Defaulted to zero for lack of listed eligibility facts, never applying the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to an adult filer with gross income ($312) under $20,000."
-us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Treated earned income as the only basis for an Oklahoma refundable credit, missing the Sales Tax Relief Credit. That credit has no earned-income requirement and pays $40 per exemption at gross income of $312."
-us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Required earned income or a qualifying dependent, but the Sales Tax Relief Credit requires neither. It pays $40 per exemption when gross income ($312) is under $20,000."
-us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Required earned income, dependents, or other special facts, missing the Sales Tax Relief Credit. For an adult filer that credit depends only on gross income under $20,000, and $312 yields $40."
-us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Found no listed facts establishing eligibility, overlooking that the $312 of gross income alone qualifies the head for the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption."
-us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance toward the Sales Tax Relief Credit income limit and denied the credit. The credit's gross income is the $312 of interest only, under $20,000, so the head receives $40."
-us,scenario_029,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Tied Oklahoma refundable credits to children or earned income and never applied the Sales Tax Relief Credit. That credit pays a single adult $40 per exemption when gross income ($312) is under $20,000."
-us,scenario_029,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Required earnings or qualifying children, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income under $20,000; here gross income is $312."
-us,scenario_029,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Required earned income, dependents, or specific expenses, overlooking the Sales Tax Relief Credit. That credit needs none of these and pays $40 for one exemption at gross income of $312."
-us,scenario_029,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Treated qualifying children as necessary for an Oklahoma refundable credit, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless adult with gross income ($312) under $20,000."
-us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Put the household over the sales tax relief income limit by counting the $50,000 of financial assistance and investment assets. The credit uses gross income of only $312 and has no asset test, so it pays $40 for one exemption."
-us,scenario_029,state_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Measured 'gross household resources' including the $50,000 of financial assistance against the sales tax relief limit. The credit's gross income is only the $312 of interest, under $20,000, which yields $40."
-us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded no refundable state credits apply, missing the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit."
-us,scenario_029,state_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Said income and assets rule out a refundable low-income credit, but the Sales Tax Relief Credit has no asset test and its gross income here is only the $312 of interest. That is under $20,000, so the credit is $40."
-us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"Treated being under 65 with no disability or dependents as disqualifying for the Sales Tax Relief Credit. Those factors only raise the limit to $50,000; an adult single filer with $312 of gross income qualifies under the base $20,000 limit for $40."
-us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit, exceeding the $20,000 under-65 limit. The credit's gross income is $312 of interest, so the head receives $40 for one exemption."
-us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Falsely stated that Oklahoma has no state income tax. Oklahoma levies an income tax, and its refundable Sales Tax Relief Credit pays $40 per exemption to this household with $312 of gross income."
-us,scenario_029,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Named the Sales Tax Relief Credit but wrongly grouped it with the EITC as needing earned income. The credit is based only on gross income under $20,000, and at $312 it pays $40 for one exemption."
-us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Considered only the EITC and child-related credits, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless adult with gross income ($312) under $20,000."
-us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Reported no indicated refundable credits, overlooking the Oklahoma Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under the $20,000 limit."
+us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Correctly identified the $40-per-exemption Sales Tax Relief Credit and its $20,000 limit, but added the $50,000 of financial assistance to gross household income. Gross income for the credit is only the $312 of taxable interest, which is far below $20,000, so one exemption yields $40."
+us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Wrongly claimed that Oklahoma has no refundable individual credits, which misses the refundable Sales Tax Relief Credit. That credit pays $40 per exemption to this filer, whose gross income of $312 is under the $20,000 limit."
+us,scenario_029,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Treated earned income as a requirement for every Oklahoma refundable credit. The Sales Tax Relief Credit does not require earned income, and it pays $40 per exemption when gross income ($312 here) is at or below $20,000."
+us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Disqualified the filer from the Sales Tax Relief Credit by counting the $50,000 of financial assistance (and citing assets) against the $20,000 income limit. The credit's gross income is only the $312 of interest, and the credit has no asset test, so the filer gets $40 for one exemption."
+us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Named the Sales Tax Relief Credit but denied it because there was no earned income and the filer had large assets and financial assistance. The credit has no earned-income or asset test, and the filer's $312 gross income is under $20,000, so it pays $40."
+us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Assumed take-up of the Sales Tax Relief Credit was zero and that its thresholds were not met, even though the prompt says to assume take-up. With $312 of gross income, the filer is under the $20,000 limit and receives $40 for one exemption."
+us,scenario_029,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,Assumed Oklahoma refundable credits require earned income or dependents. The Sales Tax Relief Credit requires neither and pays $40 per exemption to a filer with $312 of gross income.
+us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Checked only the Oklahoma EITC and never considered the refundable Sales Tax Relief Credit. That credit pays $40 per exemption because the filer's gross income ($312) is below $20,000."
+us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Wrongly asserted that Oklahoma has no general refundable credit for this filer, assuming relief requires elderly, disabled, or dependent status. Those statuses only raise the income limit to $50,000. Under the base $20,000 limit, the Sales Tax Relief Credit pays $40 on $312 of gross income."
+us,scenario_029,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable credits apply at this income level. The very low $312 income is exactly what qualifies the filer for the $40-per-exemption Sales Tax Relief Credit under its $20,000 limit."
+us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,Treated zero tax liability as ruling out refundable credits. The Sales Tax Relief Credit is refundable regardless of liability and pays $40 to a single filer with $312 of gross income.
+us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,Considered only the Oklahoma EITC (5% of a $0 federal EITC) and wrongly said no other refundable credits apply. The Sales Tax Relief Credit gives $40 for the filer's one exemption.
+us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Looked only at the EITC-based credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption when gross income ($312) is at or below $20,000."
+us,scenario_029,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,Tied all Oklahoma refundable credits to earned income and the federal EITC. The Sales Tax Relief Credit has no earned-income requirement and yields $40 here.
+us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Computed gross household income as $50,312 by adding the $50,000 of financial assistance. The credit's gross income is only the $312 of interest, which is under the $20,000 limit, so the credit is $40."
+us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Wrongly asserted that no state refundable credits apply and missed the Sales Tax Relief Credit. That credit pays $40 per exemption because gross income of $312 is below $20,000."
+us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit test. Gross income is only the $312 of interest, which is under $20,000, so the credit is $40 for one exemption."
+us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Wrongly concluded that the household qualifies for no refundable credits, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption to filers with gross income of $20,000 or less; this filer has $312."
+us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Gave $0 without any derivation and never applied the Sales Tax Relief Credit. That credit pays $40 for one exemption because gross income of $312 is under the $20,000 limit."
+us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"Wrongly believed the Sales Tax Relief Credit requires the filer to be 65+, disabled, or have dependents. Those conditions only raise the limit to $50,000; every filer qualifies under the base $20,000 limit, and the filer's $312 income meets it for $40."
+us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable Oklahoma credit applies and missed the Sales Tax Relief Credit. That credit gives $40 per exemption when gross income ($312) is at or below $20,000."
+us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Checked only the Oklahoma EITC (zero because the federal EITC is zero) and ignored the Sales Tax Relief Credit, which pays $40 for this filer's one exemption."
+us,scenario_029,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"Wrongly asserted that Oklahoma offers no refundable credit to this household. The Sales Tax Relief Credit pays $40 per exemption because gross income of $312 is below $20,000."
+us,scenario_029,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"Gave $0 without considering the Sales Tax Relief Credit. That credit pays $40 for this single filer's one exemption, given gross income of $312."
+us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Wrongly concluded that the facts support no refundable credit. The listed $312 of interest income, below the $20,000 limit, establishes eligibility for the $40 Sales Tax Relief Credit."
+us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to $0 for lack of explicit eligibility indicators. The Sales Tax Relief Credit's only conditions are an adult filer and gross income under $20,000, both met here, and it yields $40."
+us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,Assumed earned income was the basis for any Oklahoma refundable credit. The Sales Tax Relief Credit pays $40 per exemption on gross income of $312 regardless of earnings.
+us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Required earned income or a dependent for an Oklahoma refundable credit. The Sales Tax Relief Credit needs neither and gives $40 because gross income is below $20,000."
+us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,Required earned income or dependents for a refundable credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a single filer with $312 of gross income.
+us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Wrongly concluded that no facts establish eligibility. The filer's adult age and $312 of gross income (under $20,000) qualify them for the $40 Sales Tax Relief Credit."
+us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance toward the Sales Tax Relief Credit's income limit. Gross income for the credit is only the $312 of interest, so the filer qualifies for $40."
+us,scenario_029,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"Required children or earned income for a refundable credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income of $20,000 or less."
+us,scenario_029,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"Required earnings or children for eligibility and ignored the Sales Tax Relief Credit. That credit pays $40 for one exemption because gross income ($312) is below $20,000."
+us,scenario_029,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Required earned income, dependents, or specific expenses for a refundable credit. The Sales Tax Relief Credit depends only on gross income at or below $20,000 and pays $40 here."
+us,scenario_029,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,Assumed qualifying children were needed and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless single filer with $312 of gross income.
+us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance and investment assets against the Sales Tax Relief Credit limits. The credit has no asset test, and its gross income is only the $312 of interest, so the filer gets $40."
+us,scenario_029,state_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit. Gross income is $312, which is below the $20,000 limit, so the credit is $40."
+us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Wrongly asserted that no refundable state credits apply, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption because gross income ($312) is under $20,000."
+us,scenario_029,state_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Concluded that income or assets block the Oklahoma low-income credit. The Sales Tax Relief Credit has no asset test, and the filer's gross income of $312 is far below the $20,000 limit, giving $40."
+us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"Treated being under 65 with no disability or dependents as disqualifying for the Sales Tax Relief Credit. Those factors only raise the limit to $50,000; under the base $20,000 limit, the filer's $312 income qualifies for $40."
+us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Added the $50,000 of financial assistance to gross household income and found the filer over the sales-tax-relief limit. Counted gross income is only the $312 of interest, so the credit is $40 for one exemption."
+us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Falsely claimed that Oklahoma has no state income tax. Oklahoma levies an income tax and has a refundable Sales Tax Relief Credit, which pays this filer $40 on $312 of gross income."
+us,scenario_029,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"Named the Sales Tax Relief Credit but treated it as requiring earned income. It requires only gross income at or below $20,000, which $312 meets, so it pays $40."
+us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,Considered only the EITC and child-related credits and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a single filer whose gross income is $312.
+us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable credits are indicated. The filer's $312 gross income, below the $20,000 limit, qualifies them for the $40-per-exemption Sales Tax Relief Credit."
us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $5,000 of educational assistance and $12,000 of financial assistance in AGI, inflating it from $13,000 to $30,000. With only $13,000 of taxable wages, the single-filer standard deduction eliminates all taxable income, so applying the 10% and 12% brackets to $14,600 was erroneous."
us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without first applying the minimum-age requirement. At age 23, the filer is ineligible for the 2026 childless EITC, so the phase-in and phaseout arithmetic never applies."
us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated the 23-year-old as eligible for the childless EITC despite the minimum-age requirement. It also mislabeled $1,940 as the childless maximum and submitted that amount even though its own 7.65% calculation produced $994.50."
@@ -2694,52 +2780,53 @@ us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu
us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50."
us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but then used the 130% FPL gross test. It never applied Texas's broad-based categorical eligibility at 165% FPL (~$2,152/month), which this household passes at 160% FPL. As an eligible 1-person household, it gets the $24 monthly minimum allotment."
-us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It correctly got net income of about $1,658, but it then applied the net income test, which Texas broad-based categorical eligibility waives. It also turned a 30% contribution above the $298 maximum into $0 instead of the $24 minimum allotment that an eligible 1-person household receives."
-us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It counted a $30,000 total that includes the excluded $5,000 educational assistance, and it compared income to a 130% gross limit. It missed Texas's 165% FPL categorical eligibility, which countable gross income of $2,083/month passes, and the $24 minimum allotment that follows."
-us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly counted $2,083/month gross and raised Texas BBCE, but it put the 165% FPL limit at ~$2,072 when the actual figure is $2,151.88 (165% of $1,304.17). That made it wrongly fail the household. Once categorically eligible, the household gets the $24 minimum allotment even though its contribution is more than the maximum."
-us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It excluded the $12,000 of financial assistance, which SNAP counts as $1,000/month of unearned income. That left net income far too low. It then dropped its own ~$99/month result and submitted $2,298, which comes from neither its formula nor the maximum. Counting the assistance gives net income of $1,658, a $498 contribution, and the $24 minimum."
-us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It counted only the $13,000 of wages and left out the $12,000 of financial assistance, which is countable unearned income. That understated net income, which is actually $1,658/month. Once the assistance is counted, the $498 contribution is more than the $298 maximum, so the benefit is the $24 minimum rather than $134/month."
-us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It explicitly treated the $12,000 of financial assistance as non-countable, which gave net income of $657.67 and a $100.70 benefit. SNAP counts the assistance as $1,000/month of unearned income, so net income is $1,658 and the contribution is $498. That exceeds the $298 maximum, leaving the $24 minimum allotment."
-us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but stated that Texas has no broad-based categorical eligibility. Texas grants categorical eligibility through TANF non-cash benefits up to 165% FPL, and this household is at 160%, so it is eligible and gets the $24 minimum allotment."
-us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"It left the $12,000 of financial assistance out of countable income and then made an arithmetic error: $298 minus ~$200 is ~$98, not $350. Counting the assistance gives net income of $1,658 and a $498 contribution, so the benefit is the $24 minimum."
-us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the financial assistance as excluded and counted wages only, which gave net income of ~$658 and ~$101/month. SNAP counts the $1,000/month of financial assistance as unearned income, which raises net income to $1,658 and the contribution to $498. The benefit is then the $24 minimum allotment."
-us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, reaching $2,500/month gross, and it applied a 130% gross test. The correct countable gross is $2,083/month, which passes Texas's 165% FPL categorical eligibility limit. The household therefore gets the $24 minimum allotment."
-us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It counted a $30,000 total that includes the excluded educational assistance, and it applied the 130% FPL gross limit. It ignored Texas's 165% FPL broad-based categorical eligibility, which the correct $25,000 countable income passes, and the $24 monthly minimum allotment that follows."
-us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It wrongly counted the $5,000 of educational assistance, and it treated a contribution above the one-person maximum as a $0 benefit. A categorically eligible 1-person household whose contribution exceeds the $298 maximum still receives the $24 minimum allotment."
-us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It counted the pregnant head as a household of 2 and used the $546 two-person maximum. SNAP does not count an unborn child, so the unit is 1 person with a $298 maximum. Its correct $497 contribution exceeds that maximum, so the benefit is the $24 minimum allotment."
-us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It correctly used a household of one and $25,000 of countable income, but it applied the 130% FPL gross test. It missed Texas's 165% FPL broad-based categorical eligibility (~$25,823/year), which the household passes, and so it missed the $24 monthly minimum allotment."
-us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It based the benefit on $13,000 of earnings alone and left out the $12,000 of financial assistance, which is countable unearned income. Its $217/month result also does not follow from the SNAP formula. Correct net income is $1,658, so the contribution is more than the maximum and the benefit is the $24 minimum."
-us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test even though Texas broad-based categorical eligibility replaces it with a 165% FPL limit. The household's 160%-of-FPL gross income passes that limit, which entitles it to the $24 minimum allotment."
-us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It correctly raised the 165% FPL BBCE limit but calculated it from the outdated $15,060 guideline, getting $24,849. The guideline in effect is $15,650, which makes the limit $25,822.50, so $25,000 passes. The household is categorically eligible for the $24 minimum allotment."
-us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without any computation. This household is categorically eligible under Texas BBCE (gross at 160% FPL, $190 in assets) and has a $498 contribution that exceeds the $298 maximum. Such a household receives the $24 monthly minimum allotment, not $0."
-us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It applied net income limits that Texas broad-based categorical eligibility waives, and it treated a formula benefit of zero as no benefit. An eligible 1-person household whose contribution exceeds the maximum receives the $24 minimum allotment."
-us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It applied a net income threshold that categorical eligibility waives for this household, whose gross income is 160% FPL, under Texas's 165% limit. It also missed the $24 minimum allotment for eligible 1-person households whose computed benefit is zero."
-us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance as unearned income but turned a formula benefit of zero into $0. The household is categorically eligible, so the $24 minimum allotment for 1-person households applies."
-us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"It counted the pregnancy as a second household member and allowed an $8,080 medical deduction. SNAP excludes unborn children from household size, and it allows the medical deduction only for elderly or disabled members. With 1 person and no medical deduction, net income is $1,658, the contribution exceeds the $298 maximum, and the benefit is the $24 minimum."
-us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"It counted income correctly but applied the standard gross and net limits, which Texas broad-based categorical eligibility replaces with a 165% FPL gross test. The household passes that test and receives the $24 minimum allotment."
-us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated the household as having very low countable income, which means it left out the $12,000 of financial assistance that SNAP counts as unearned income. Once it is counted, net income is $1,658 and the contribution is $498, so the benefit is the $24 minimum rather than ~$176/month."
-us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying any SNAP test. The household is categorically eligible under Texas BBCE at 160% FPL, and an eligible 1-person household with a contribution above the maximum receives the $24 monthly minimum allotment."
-us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the $298 one-person maximum, but it concluded that no benefit is payable. The household is categorically eligible, so the $24 minimum allotment for 1- and 2-person households applies."
-us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It used a one-person gross income limit without applying Texas's 165% FPL broad-based categorical eligibility threshold, which the household passes at 160% FPL. It therefore missed the $24 monthly minimum allotment."
-us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It correctly counted $25,000 and excluded the educational assistance, but it wrongly concluded that this exceeds Texas's gross limit. The Texas BBCE limit is 165% FPL ($25,822.50), so the household is eligible and gets the $24 minimum allotment."
-us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It tested $25,000 against the standard one-person gross limit (130% FPL) instead of Texas's 165% FPL broad-based categorical eligibility limit, which the household passes. It then missed the $24 minimum allotment."
-us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,It applied SNAP income limits that Texas broad-based categorical eligibility waives for households at or below 165% FPL; this household is at 160%. It did not apply the $24 minimum allotment for an eligible 1-person household.
-us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance and found that the 30% contribution exceeds the one-person maximum, but it returned $0. The categorically eligible household instead receives the $24 minimum allotment each month."
-us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It based net income on wages alone and left out the $1,000/month of financial assistance, which is countable unearned income. Including it raises net income to $1,658 and the contribution to $498, so the benefit drops to the $24 minimum ($288/year)."
-us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,It correctly counted wages and financial assistance but treated a formula allotment of zero as no benefit. A categorically eligible 1-person household receives the $24 minimum allotment.
-us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 without a computation. The household passes Texas BBCE at 160% FPL with $190 in assets, and its $498 contribution exceeds the $298 maximum, which triggers the $24 minimum allotment."
-us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test to $25,000 without recognizing that Texas broad-based categorical eligibility raises the limit to 165% FPL ($25,822.50). The household passes that limit and receives the $24 minimum allotment."
-us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"It claimed the income exceeds the Texas BBCE 165% FPL limit, but countable income of $25,000 (wages plus financial assistance, with educational assistance excluded) is below $25,822.50. The household is categorically eligible and receives the $24 minimum allotment."
-us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"It correctly computed net income of ~$19,900/year ($1,658/month) but failed the household on the 100% FPL net income test. Texas broad-based categorical eligibility waives that test, so the household is eligible and receives the $24 minimum allotment."
-us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It correctly counted $25,000 but applied the 130% FPL gross limit instead of Texas's 165% FPL broad-based categorical eligibility limit, which the household passes. It therefore missed the $24 monthly minimum allotment."
-us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and applied the 130% FPL gross test. Correct countable gross income of $2,083/month passes Texas's 165% FPL categorical eligibility limit, so the household gets the $24 minimum."
-us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable SNAP value or explanation, so the requested output is missing."
-us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly computed $2,083.33/month but compared it to the standard one-person gross limit instead of Texas's 165% FPL BBCE limit ($2,151.88). It therefore missed eligibility and the $24 minimum allotment."
-us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"It left out the $12,000 of countable financial assistance and compared an annual net figure ($10,400) with a monthly limit ($1,255), mixing time periods. With correct monthly countable income under Texas BBCE, the household is eligible and its $498 contribution produces the $24 minimum allotment."
-us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"It correctly counted $25,000 but applied the 130% FPL gross test (~$20,350). Texas broad-based categorical eligibility uses a 165% FPL limit ($25,822.50), which the household passes, so it receives the $24 minimum allotment."
-us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It left out the $12,000 of financial assistance and the standard deduction, and it then submitted $2,576, which contradicts its own $31–32/month calculation. Counting the assistance gives net income of $1,658, a $498 contribution, and the $24 minimum allotment."
-us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It counted the unborn child to make a 2-person household and assumed $0 net income despite $2,083/month of countable wages and financial assistance. The unit is 1 person with a $298 maximum and a $498 contribution, so the benefit is the $24 minimum. Its submitted $3,576 also contradicts its own $4,272."
+us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but tested it against the regular 130% FPL gross limit. It never applied Texas's broad-based categorical eligibility (BBCE) limit of 165% FPL ($2,152/month), which the household passes. As a result it missed the $24/month minimum allotment that a categorically eligible one-person household receives."
+us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It correctly computed net income of about $1,658 and found that 30% of net income exceeds the $298 maximum. However, it applied a net income test that BBCE waives in Texas, and it set the benefit to $0 instead of the $24/month minimum allotment for a categorically eligible one-person household."
+us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It added the $5,000 of educational assistance to income, but SNAP excludes it, so the correct total is $25,000 rather than $30,000. It then applied only a 130% FPL gross limit and missed Texas's 165% BBCE limit, which the actual $2,083/month gross income passes. That would have led to the $24/month minimum allotment."
+us,scenario_030,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It used only wages and returned $0 without applying any rule. It left out Texas's BBCE pathway (gross income of $2,083 is below the 165% FPL limit of $2,152) and the $24/month minimum allotment for an eligible one-person household."
+us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It identified Texas BBCE at 165% FPL but calculated the limit as about $2,072 using the outdated 2025 guideline. The 2026 limit is 165% × $1,304.17 = $2,152, which the $2,083 gross income passes. Categorical eligibility then yields the $24/month minimum allotment, not $0."
+us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It excluded the $12,000 of financial assistance, which SNAP counts as $1,000/month of unearned income, so it understated net income at $663 instead of $1,658. It also abandoned its own $99/month result and jumped to a near-maximum figure. The correct contribution of $498 exceeds the $298 maximum, which leaves only the $24 minimum."
+us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It counted only the $13,000 of wages and left out the $1,000/month of financial assistance that SNAP counts as unearned income. That produced a $134/month benefit. With the assistance included, net income is $1,658, the 30% contribution ($498) exceeds the $298 maximum, and the household receives only the $24 minimum allotment."
+us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It assumed the financial assistance does not count, but SNAP counts the $12,000 of financial assistance as unearned income. Only the educational assistance is excluded. Including the $1,000/month raises net income to $1,658 and the contribution to $498, so the benefit falls to the $24 minimum rather than $101/month."
+us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It said Texas does NOT have broad-based categorical eligibility, which is wrong. Texas grants categorical eligibility through TANF non-cash services at a 165% FPL gross limit ($2,152/month), and the $2,083 gross income passes it. That waives the income tests and pays the $24/month minimum allotment."
+us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"It left out the $1,000/month of financial assistance, which counts as unearned income. It also made an arithmetic error: $298 minus about $200 is roughly $98, not $350, and $350/month exceeds the $298 maximum allotment. With all income counted, the contribution of $498 leaves only the $24 minimum."
+us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the financial assistance as excluded, but SNAP counts the $12,000 as unearned income. Only the educational assistance is excluded. Including it gives net income of $1,658 and a $498 contribution above the $298 maximum, so the benefit is the $24 minimum allotment rather than $101/month."
+us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and so inflated gross income to $2,500/month instead of $2,083. It also applied the 130% gross limit instead of Texas's 165% BBCE limit of $2,152, which the correct income passes. That would have led to the $24/month minimum allotment."
+us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It counted the excluded $5,000 of educational assistance, giving gross income of $30,000 instead of $25,000, and compared it to a 2025-based 130% FPL limit. Texas BBCE uses 165% of the 2026 guideline ($25,822/year), which $25,000 passes. That makes the household categorically eligible for the $24/month minimum."
+us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, so its gross income exceeded even the 165% BBCE limit. It also zeroed the benefit when the contribution exceeded the maximum. With correct income of $2,083/month the household is categorically eligible, and the $24/month minimum allotment applies."
+us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It counted the pregnant head as a two-person household and used the $546 two-person maximum. SNAP does not count an unborn child, so the household has one member and the maximum is $298. Its $497 contribution exceeds $298, which leaves only the $24 minimum allotment ($288/year)."
+us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It correctly used a one-person household with $25,000 of countable income but tested it only against the 130% FPL limit. Texas BBCE raises the gross limit to 165% FPL ($25,822/year), which the household passes. Categorical eligibility then pays the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It based the benefit only on the $13,000 of wages and left out the $1,000/month of financial assistance that counts as unearned income. Correct net income is $1,658, which gives a $498 contribution above the $298 maximum. That leaves only the $24 minimum, not about $217/month."
+us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income limit and missed Texas's BBCE limit of 165% FPL ($2,152/month), which the $2,083 gross income passes. Categorical eligibility waives the income tests and pays the $24/month one-person minimum allotment."
+us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It applied the 165% FPL BBCE limit but calculated it as $24,849 using the 2025 guideline of $15,060. The 2026 guideline is $15,650, which makes the limit $25,822.50, so the $25,000 income passes. The household is then categorically eligible for the $24/month minimum."
+us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without any derivation. Under Texas BBCE (gross income $2,083 is below the 165% FPL limit of $2,152), the household is categorically eligible. Although its $498 contribution exceeds the $298 maximum, it receives the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It treated income above the net limit as disqualifying, but Texas BBCE waives the net income test. When the contribution exceeds the maximum allotment, a categorically eligible one-person household still receives the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,It applied a net income threshold that Texas BBCE waives and returned $0. It missed that a categorically eligible one-person household whose $498 contribution exceeds the $298 maximum receives the $24/month minimum allotment.
+us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance as unearned income. However, it applied a net income threshold that BBCE waives and dropped the benefit to $0 instead of paying the $24/month minimum allotment that applies to categorically eligible one- and two-person households."
+us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"It counted the unborn child to make a household of two, but SNAP counts only the pregnant head. It also deducted $8,080 of medical expenses, although the excess medical deduction applies only to elderly or disabled members. With one person and no medical deduction, the contribution of $498 exceeds the $298 maximum, which leaves the $24 minimum."
+us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"It counted income correctly but applied the regular gross and net income limits. Texas BBCE uses a 165% FPL gross limit ($2,152/month), which the $2,083 income passes, and it waives the net test. The household therefore receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated countable income as very low, ignoring $1,083/month of wages and $1,000/month of financial assistance. Correct net income is $1,658, and the $498 contribution exceeds the $298 maximum, so only the $24 minimum allotment is paid."
+us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying any SNAP rule. The household is categorically eligible under Texas BBCE because gross income of $2,083 is below the 165% FPL limit of $2,152. With a contribution above the maximum allotment, it receives the $24/month one-person minimum."
+us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,It correctly found that 30% of net income exceeds the $298 one-person maximum. It then set the benefit to $0 and missed that a household categorically eligible through Texas BBCE receives the $24/month minimum allotment.
+us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It ruled the household out on the gross income limit and missed that Texas BBCE sets that limit at 165% FPL ($2,152/month). The $2,083 gross income passes it, so the household is categorically eligible and receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It said $25,000 exceeds the Texas gross income limit, but the Texas BBCE limit is 165% of the 2026 guideline ($25,822.50/year), which the household passes. Categorical eligibility then pays the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It compared $25,000 to the regular gross income limit and missed Texas BBCE at 165% FPL ($25,822.50/year). Under BBCE the household is categorically eligible and receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"It applied the standard SNAP income limits, which Texas BBCE replaces with a 165% FPL gross test that this household passes. It missed the $24/month minimum allotment that applies when the $498 contribution exceeds the $298 maximum."
+us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,It correctly counted the financial assistance and found that the 30% contribution exceeds the $298 maximum. It then returned $0 instead of the $24/month minimum allotment owed to a categorically eligible one-person household.
+us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It deducted 30% of net wages only and left out the $1,000/month of financial assistance that SNAP counts as unearned income. Correct net income is $1,658, which gives a $498 contribution above the $298 maximum and leaves only the $24 minimum allotment."
+us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,It correctly found countable income too high for a positive formula benefit but ignored the minimum allotment. A one-person household categorically eligible through Texas BBCE receives $24/month even when the contribution exceeds the maximum.
+us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 without any computation. The household is categorically eligible under Texas BBCE (gross income of $2,083 is below the 165% FPL limit of $2,152) and receives the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross limit to the $25,000 of countable income and missed Texas BBCE at 165% FPL ($25,822.50/year). The household passes that limit and receives the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"It claimed income is above the Texas 165% FPL BBCE limit, but $25,000 is below 165% × $15,650 = $25,822.50. That makes the household categorically eligible and entitles it to the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"It disqualified the household on the 100% FPL net income test, which Texas BBCE waives for categorically eligible households. It also missed that the household receives the $24/month minimum allotment when the contribution exceeds the maximum."
+us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross limit (about $20,800) and missed Texas BBCE, which raises the limit to 165% FPL ($25,822.50/year). The $25,000 income passes that limit, which makes the household eligible for the $24/month minimum allotment."
+us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and tested income against the 130% FPL limit. Correct gross income is $2,083/month, which passes Texas's 165% BBCE limit of $2,152 and yields the $24/month minimum allotment."
+us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there is no answer to score. The correct derivation is BBCE categorical eligibility with a $24/month minimum allotment, or $288/year."
+us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly averaged countable income at $2,083.33/month but applied the regular gross income limit. Texas BBCE sets the limit at 165% FPL ($2,152/month), so the household is categorically eligible for the $24/month minimum allotment."
+us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"It compared annual net income of $10,400 against a monthly limit of $1,255, which is a unit mismatch. It also left out the $1,000/month of financial assistance. Correct monthly net income of $1,658 under Texas BBCE gives a $498 contribution above the maximum and a $24/month minimum allotment."
+us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"It correctly counted $25,000 of income but tested it only against the 130% FPL limit (about $20,350). Texas BBCE uses 165% FPL ($25,822.50/year), which the household passes, and that results in the $24/month minimum allotment."
+us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It counted only wages, leaving out the $1,000/month of financial assistance, and skipped the $209 standard deduction. It then dropped its own $31–32/month result and gave an unsupported $2,576. Correct net income is $1,658, which gives a contribution above the $298 maximum and the $24 minimum."
+us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It counted the unborn child as a second household member and set net income to $0, ignoring $1,083/month of wages and $1,000/month of financial assistance. The SNAP unit is one person with net income of $1,658, so the benefit is the $24/month minimum, not a maximum allotment."
us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation."
us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply."
@@ -2747,49 +2834,50 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er
us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income."
us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation."
us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic."
-us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took the aged pathway and a 138% limit of about $1,800 a month, subtracted the $1,165 of alimony to put income at about $1,890 a month, and assumed no specific disregard, saying even SSI-style disregards would leave it over. The engine does not subtract alimony; it takes $1,987.79 a month less California's $230 disregard, which replaces SSI's $20 exclusion, for $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the aged pathway but applied only SSI's $240 annual general exclusion, putting countable income at $23,613 against a 138% limit of about $21,600. California's $230 monthly disregard takes the place of that $20 exclusion, so the engine counts $23,853.47 less $2,760, or $21,093.47, which is under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853, and that amount less the $1,165 of alimony, against a 138% limit it understated at about $18,564, and doubted the $4,200 in assets. The engine subtracts no alimony: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline). The $4,200 is far under California's $130,000 asset limit."
-us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It first called gross income of $23,853 well below the threshold, then switched to a MAGI-based 100% limit of about $15,650 (the 2025 guideline), and its explanation concluded the head was ineligible. The engine's senior pathway uses SSI income rules and 138% of the 2026 guideline, $22,024.80; it counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against an aged limit it gave as 100% to 138% of the poverty line, about $15,650 to $21,600 (2025-guideline figures), and found it over. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the 138% aged limit, at about $21,600 (the 2025-guideline figure), but compared it with gross income of $23,853. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under its $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It moved between the adult expansion test, which excludes someone aged 67, and a 100% aged limit of $15,060; its explanation settled on the expansion test, with gross income of $23,853 against $20,783, which is 138% of its understated $15,060 guideline. The engine's senior pathway counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under $22,024.80 (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded most Social Security to get $14,408, said 138% applied only to expansion adults under 65, cited a 138% aged limit of about $21,597 that $14,408 would pass, then used an SSI limit of about $13,596 as binding. The engine counts Social Security in full and uses 138% of the 2026 guideline, $22,024.80: $23,853.47 less California's $230 monthly disregard ($2,760 a year) gives $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head was in no categorical group and that countable income of about $23.9k, including Social Security and with no disregard, exceeded 138% FPL. At 67 the head is aged, which alone meets the category condition of the engine's senior-or-disabled pathway, and the engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It compared gross income of about $23,853 with 138% FPL and concluded the head was over the limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It tested gross monthly income of $1,987.75 against an understated 138% FPL limit of $1,732. The engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $1,757.79, under the actual limit of $1,835.40 a month (138% of the 2026 guideline). Both errors mattered: income after the disregard still tops the model's $1,732 figure."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable income and found it over California's FPL-based limit for a senior, without stating the limit. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with its own 138% FPL figure of $22,218, slightly above the engine's $22,024.80 limit (138% of the 2026 guideline). The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under either figure."
-us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a 138% FPL limit of about $22,080, close to the engine's $22,024.80 (138% of the 2026 guideline), but tested gross income of $23,853 against it. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves countable income of $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no figures or pathway, saying only that household income exceeded the Medicaid limit ""for a single adult."" At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 138% FPL, without stating the dollar limit, and found it over. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It correctly picked the 138% FPL limit for what it called ""free non-MAGI Medi-Cal"" but gave no figures, saying only that the head's monthly income was over it; in the same response it put gross income at $23,853 for its federal tax and SNAP answers. The engine subtracts California's $230 monthly disregard from $1,987.79 a month, in place of SSI's $20 exclusion. That leaves $1,757.79 a month, under the $1,835.40 monthly limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no figures, threshold or pathway, citing only the head's age and income. At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) is $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable and said it exceeded an unstated threshold for people 65 or older. Gross income is $1,828.67 over the limit, but the engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It placed the head in what it called the ""non-MAGI"" elderly pathway but gave no income figure or limit, and it judged income too high, which holds against the engine's limit only if California's $230 disregard is skipped or replaced by a smaller one, such as SSI's $20. In the engine's senior-or-disabled pathway, $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It tested total income against a 138% FPL limit with no disregard and called it ""well"" over, but $23,853.47 is about 149% of the 2026 guideline, only $1,828.67 over the $22,024.80 limit. The engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed that Medicare eligibility at 65 or older bars Medicaid and ran no income test. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group, and age 67 alone meets the aged test for the senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It applied the adult expansion limit, which excludes anyone over 64, comparing gross income of about $1,988 a month (about 149% of the 2026 guideline, not the 199% it stated) with an understated limit of about $1,600. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $1,987.79 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It denied eligibility because no disability or low-income flag was listed and income was ""substantial"", without comparing any income figure with a limit. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It correctly ruled out the adult expansion group at age 67 but tested the full $23,853 against an unstated aged income or ""spend-down"" standard, with only minimal medical expenses to offset it. The engine's senior-or-disabled pathway instead subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no figures, pathway or threshold and only asserted that countable income was above the applicable California limit, which holds against the engine's limit only if the state's $230 monthly disregard ($2,760 a year) is skipped or replaced by a smaller one, such as SSI's $20. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $23,853.47 less that disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It identified the aged pathway but concluded, without showing figures, that retirement income exceeded California's limit. That holds against the engine's limit only if California's $230 monthly disregard is skipped or replaced by a smaller one, such as SSI's $20: gross income of $23,853.47 is $1,828.67 over the limit. The engine's senior-or-disabled pathway subtracts that disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It placed the head in the aged pathway but tested income against an ""SSI-linked income level,"" giving no figures. The engine's limit for this pathway in California is $22,024.80 (138% of the 2026 poverty guideline), and countable income is $21,093.47: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion."
-us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It correctly ruled out the adult expansion group at 67 but said retirement income after the general income exclusion exceeded California's aged limit, giving no figures. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline)."
-us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It said the head's retirement income exceeded California's aged Medicaid threshold, naming no exclusion and giving no figures. Gross income of $23,853.47 is over the $22,024.80 limit (138% of the 2026 poverty guideline), but the engine first subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving countable income of $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It said retirement income exceeded California's aged limit after ""the applicable general income exclusion,"" without stating its amount, the income or the limit. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline)."
-us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It ruled the head out as Medicare-eligible, claiming the engine excludes ""full dual eligibles"" from Medicaid. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group: at 67 the head counts as aged for the senior-or-disabled pathway, countable income of $21,093.47 ($23,853.47 less California's $230 monthly disregard, $2,760 a year) is under the $22,024.80 limit (138% of the 2026 poverty guideline), and $4,200 in assets is under the $130,000 limit."
-us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It said bank assets of $4,200 and countable income of about $23,853 both exceeded California's aged limits, counting gross income in full. The pathway's asset limit is $130,000, so $4,200 passes, and the engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline)."
-us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It counted the full $23,853 as ""MAGI/ABD countable income"" and said it exceeded California's ""Aged & Disabled FPL"" limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline)."
-us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied only the $20 monthly unearned-income disregard before testing against California's 138% FPL limit, which leaves about $23,613, still over the $22,024.80 limit. The engine uses California's $230 monthly disregard in place of that $20 exclusion, so countable income is $21,093.47 ($23,853.47 less $2,760), under the limit."
-us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied a $2,000 asset limit to aged Medicaid and denied eligibility because the $4,200 in bank assets exceeded it, but $2,000 is SSI's resource limit, not this pathway's test. The engine's California senior-or-disabled pathway allows assets below $130,000, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It applied a 100% FPL limit of about $15,650, the 2025 guideline, and compared gross income of about $23,853 against it. The engine uses 138% of the 2026 guideline, $22,024.80, and subtracts California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) from $23,853.47, leaving countable income of $21,093.47, under the limit."
-us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It correctly placed the head outside the adult expansion group but ruled out the aged and disabled pathways because the head was not disabled, blind or an SSI recipient. The engine's senior-or-disabled pathway needs no disability, blindness or SSI receipt; age 67 alone meets its aged criterion. Countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly placed the head outside the adult expansion group but said the listed retirement income exceeded the California aged income limit, without naming a limit or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It guessed that about $24,000 of income might exceed an unstated senior limit and answered 0 without naming a threshold or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving countable income of $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It correctly treated the head as aged but applied SSI's $2,000 resource limit to the $4,200 in bank assets and said gross income of about $24,000 exceeded the aged income standard. The engine's California pathway allows assets below $130,000 and subtracts a $230 monthly disregard ($2,760 a year) from the $23,853.47 of income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted $23,853 of Social Security, pension and IRA income in full, said it exceeded ""typical aged/blind/disabled Medicaid limits"" without naming one, and cited the missing disability or SSI status. The engine's senior-or-disabled pathway needs neither, since age 67 alone meets its aged criterion, and it subtracts California's $230 monthly disregard ($2,760 a year) from $23,853.47, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
-us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the 67-year-old was not disabled, blind or in any other listed category, and that income was above the senior pathway's limit without naming a figure. Age 65 or older alone meets the aged criterion of the engine's senior-or-disabled pathway, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)."
+us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It subtracted the $1,165 of alimony paid, which SSI-based income counting does not allow. It never applied California's $230/month Aged & Disabled income disregard. Countable income is $23,853 − $2,760 = $21,093, which is under the $22,024.80 limit (138% of the 2026 FPL)."
+us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It applied SSI's $20/month ($240/year) general exclusion instead of California's $230/month ($2,760/year) disregard, which replaces it. It also used an outdated ~$21,600 limit instead of $22,024.80. With the correct disregard, countable income is $21,093, which is under the limit."
+us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It used a wrong 138% FPL figure (~$18,564) and compared gross income without California's $230/month disregard. It also suggested that the $4,200 in savings fails a resource test, but California's limit for an individual is $130,000. Countable income of $21,093 is under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated Medicare age as ruling out Medicaid and never evaluated the aged (senior-or-disabled) pathway. Being 65 or older meets that pathway's category test, and Medicare eligibility bars only the 19–64 expansion group."
+us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the aged pathway as MAGI-based with a 100% FPL limit. California's Aged & Disabled program uses SSI-style counting with a $230/month disregard and a 138% FPL limit ($22,024.80), and countable income of $21,093 is under that limit."
+us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the limit without subtracting California's $230/month ($2,760/year) Aged & Disabled disregard. After the disregard, countable income is $21,093, under 138% of the 2026 FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to an outdated ~$21,600 limit and skipped California's $230/month disregard. The correct comparison is $21,093 of countable income against $22,024.80, which passes."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It ultimately applied the MAGI expansion pathway, which does not cover a 67-year-old. It used an understated $20,783 limit and gave no disregard. The aged non-MAGI pathway subtracts $2,760/year from $23,853 and compares the resulting $21,093 to $22,024.80, which passes."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded Social Security, which SSI-style counting includes in full. It then tested income against the SSI federal benefit rate (~$13,596) instead of the 138% FPL limit for California's Aged & Disabled program. Correct countable income is $23,853 − $2,760 = $21,093, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"It claimed the head is in no categorical group, but age 67 meets the aged category on its own. It also compared gross income to 138% FPL without California's $230/month disregard, which brings countable income to $21,093, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It tested gross income of $23,853 against 138% FPL and omitted California's $230/month Aged & Disabled disregard. Countable income after the disregard is $21,093, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It compared gross monthly income of $1,987.75 to an understated limit of $1,732; the correct limit is $1,835.40/month. It also skipped the $230/month disregard, which brings countable income to $1,757.79/month, under the limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It counted the full $23,853 as countable income and never subtracted California's $230/month disregard for the Aged & Disabled program. Countable income of $21,093 is under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a limit close to the correct one but compared gross income of $23,853 with no disregard. California subtracts $230/month ($2,760/year), leaving $21,093, which is under 138% FPL."
+us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied the correct 138% FPL test but used gross income of $23,853 and missed California's $230/month disregard. Countable income of $21,093 is under the ~$22,025 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no specific reasoning; the answer is consistent with comparing gross income to the limit. The correct derivation subtracts California's $2,760 annual Aged & Disabled disregard from $23,853, leaving $21,093, which is under the $22,024.80 limit (138% FPL)."
+us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 to 138% FPL without California's $230/month disregard. Countable income of $21,093 passes the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It identified the correct non-MAGI 138% FPL test but used gross monthly income (~$1,988) without the $230/month disregard. After the disregard, income is ~$1,758/month, under the $1,835.40 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no specific reasoning, and the answer implies the income test failed. The aged pathway is open at 67, and after California's $2,760 annual disregard, countable income of $21,093 is under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It tested gross income of $23,853 and never subtracted California's $230/month Aged & Disabled disregard. That disregard brings countable income to $21,093, under 138% FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It concluded income exceeds the non-MAGI limit without applying California's $230/month disregard. Countable income of $21,093 is under the $22,024.80 limit (138% FPL)."
+us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It compared total income to 138% FPL with no deductions and missed California's $230/month disregard. Income after the disregard is $21,093, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed that Medicare eligibility at 65 or older rules out Medicaid. In PolicyEngine, Medicare eligibility bars only the 19–64 expansion group, and the aged senior-or-disabled pathway remains open and passes here."
+us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It tested the head against the expansion group, which covers only ages 19–64, using a wrong FPL figure (its numbers put income at ~199% FPL). It never evaluated the aged pathway, where income after the $230/month disregard is $21,093, under 138% FPL."
+us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required disability or other qualifying flags and did not recognize that age 67 alone meets the aged category. It also never ran the income test, which passes after California's $230/month disregard ($21,093 vs. $22,024.80)."
+us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged limit and fell back on spend-down, omitting California's $230/month disregard. With the disregard, countable income of $21,093 meets the $22,024.80 limit (138% FPL) for full-scope eligibility."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It concluded countable income exceeds the limit without applying California's $230/month Aged & Disabled disregard. That disregard leaves $21,093, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged limit without subtracting California's $2,760 annual disregard. Countable income of $21,093 is under 138% of the 2026 FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested income against an SSI-linked income level instead of the 138% FPL limit for California's Aged & Disabled program. It also omitted the $230/month disregard; applying it gives $21,093 of countable income, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It applied SSI's $20/month general income exclusion. California substitutes a $230/month disregard for it, which leaves $21,093, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged threshold without California's $230/month disregard. After the disregard, countable income of $21,093 passes the 138% FPL limit."
+us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It used SSI's $20 general income exclusion instead of California's $230/month disregard, which replaces it. With the correct $2,760 annual disregard, countable income is $21,093, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It assumed PolicyEngine excludes Medicare-eligible people from Medicaid. Medicare eligibility bars only the 19–64 expansion group, and the aged senior-or-disabled pathway grants eligibility here."
+us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It claimed $4,200 in bank assets exceeds the resource limit, but California's Aged & Disabled limit for an individual is $130,000. It also compared gross income without the $230/month disregard, which brings countable income to $21,093, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It treated the full $23,853 as countable and missed California's $230/month Aged & Disabled disregard. Countable income of $21,093 is under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,state_local_rule,False,"It applied SSI's $20/month unearned-income disregard instead of California's $230/month disregard, which replaces it. Using $230 yields $21,093 of countable income, under 138% FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal SSI $2,000 resource limit. California's Aged & Disabled resource limit for an individual is $130,000, so $4,200 passes easily."
+us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650) and gross income. The aged pathway allows up to 138% of the 2026 FPL ($22,024.80) after California's $230/month disregard, and countable income of $21,093 passes."
+us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It held that the non-MAGI aged/disabled pathway requires disability, blindness, or SSI receipt. Age 65 or older alone meets the aged category, and the income ($21,093 after the disregard) and asset tests both pass."
+us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It compared retirement income to the aged Medi-Cal limit without California's $230/month disregard. That disregard reduces countable income to $21,093, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It guessed that roughly $24k of income exceeds the limit without doing the calculation. Subtracting California's $2,760 annual disregard gives $21,093, which is under 138% of the 2026 FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the SSI $2,000 resource limit; California's Aged & Disabled limit is $130,000. It also skipped the $230/month income disregard, after which countable income of $21,093 meets the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged limit without California's $230/month disregard and wrongly required disability or SSI status. Age 67 alone meets the aged category, and income after the disregard ($21,093) is under 138% FPL."
+us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the head fits no eligibility category, but age 67 meets the aged category on its own. It also skipped California's $230/month disregard, which brings countable income to $21,093, under the $22,024.80 limit."
us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated gross income under 200% FPL as automatic BBCE eligibility and never ran the net income test PolicyEngine applies to this elderly household. Net income of about $1,779/mo ($1,987.75 minus the $209 standard deduction, with no shelter or medical deduction) is above the roughly $1,305/mo net limit. The household is ineligible, so the $23/mo minimum benefit ($276/yr) does not apply."
us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model assumed BBCE at 200% FPL makes a 1-person household categorically eligible and entitled to the minimum benefit. It skipped the 100% FPL net income test that PolicyEngine applies to this elderly household. Net income of about $1,779/mo is above the roughly $1,305/mo limit, so the household is ineligible and gets $0, not $276."
us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated countable income as very low. In fact, all $23,853 of Social Security, pension and IRA income counts as unearned income, which is $1,987.75/mo gross. Its $144/mo ($1,728/yr) benefit only works if net income is about $513/mo, which means it left out most of the pension and Social Security income. The real net income of about $1,779/mo fails the net income test, and 30% of it ($534) is more than the $298 maximum allotment, so SNAP is $0."
@@ -2811,13 +2899,14 @@ us,scenario_032,child1_chip_eligible,grok-4.3,llm_error,health_coverage,False,"I
us,scenario_032,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for child1_chip_eligible, so the key is absent rather than substantively wrong. The correct derivation places the 6-year-old in Minnesota's OLDER_CHILD Medical Assistance category at ~160% FPL, which excludes the child from CHIP and yields 0."
us,scenario_032,child1_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"It checked only the under-19 age limit and compared $42,664 to a ~218% FPL CHIP threshold, then closed with an irrelevant Medicare statement — an age-and-income screen with no Medicaid step. Minnesota's Medical Assistance covers children 6-18 above this income, so the child is Medicaid-eligible under the OLDER_CHILD category and categorically outside CHIP."
us,scenario_032,child1_early_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_032,child1_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model correctly found income above 100% FPL but converted Head Start's limited allowance to enroll some over-income children into an individual entitlement. It also asserted categorical eligibility without identifying any qualifying categorical circumstance in the household.
-us,scenario_032,child1_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model declared categorical and income eligibility without naming a categorical pathway or applying the traced income test. Adjusted gross income is $30,916, above the $27,320 federal poverty guideline, and no categorical pathway applies."
-us,scenario_032,child1_head_start_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model reversed the income comparison by claiming countable income was below the poverty guideline. The applicable figures are $30,916 of adjusted gross income and a $27,320 guideline, so the income test fails."
-us,scenario_032,child1_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required output entirely, so it did not submit the binary Head Start eligibility value or a supporting explanation."
-us,scenario_032,child1_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated possible participation by an older child in a mixed-age or enrichment program as establishing Head Start eligibility. It never applied either required eligibility route: no categorical pathway exists, and $30,916 of adjusted gross income exceeds the $27,320 guideline."
-us,scenario_032,child1_head_start_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used $29,000 of wages instead of the traced $30,916 adjusted gross income, used an incorrect family-of-three poverty guideline, and treated 130–150% FPL as the ordinary Head Start income cutoff. The applicable test compares $30,916 with $27,320, and the household fails it."
-us,scenario_032,child1_head_start_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that income was within the Head Start limit without performing the required comparison. Adjusted gross income of $30,916 exceeds the $27,320 federal poverty guideline, and no categorical eligibility pathway applies."
+us,scenario_032,child1_head_start_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said Yes based only on the child's age and never ran the 100% FPL income test. Household AGI of $30,916 is above the $27,320 guideline for a family of three, and no categorical pathway applies, so the child is not eligible."
+us,scenario_032,child1_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly found that income is above 100% FPL, then reversed itself by citing the program-level allowance of up to 10% over-income enrollment and an unnamed 'categorical eligibility'. That allowance is a program enrollment slot, not individual eligibility. The household also receives no SSI or TANF and has no foster or homeless status, so with AGI of $30,916 above the $27,320 FPG, the answer is not eligible."
+us,scenario_032,child1_head_start_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model called the income 'low relative to family size' without comparing it to the 100% FPL limit. AGI of $30,916 ($29,000 wages plus $1,916 of taxable SS benefits) is above the $27,320 FPG, and no categorical pathway exists, so the child is not eligible."
+us,scenario_032,child1_head_start_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model claimed household income is 'well below' the poverty guideline for a family of three, which is false. AGI is $30,916, above the $27,320 FPG, so the 100% FPL income test fails and no categorical pathway applies."
+us,scenario_032,child1_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for child1_head_start_eligible. The correct answer is 0, because AGI of $30,916 is above the $27,320 poverty guideline and there is no categorical pathway."
+us,scenario_032,child1_head_start_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model based its answer on a guess that a 6-year-old can attend mixed-age programs and never applied the income test. AGI of $30,916 is above the 100% FPL threshold of $27,320 for three people, and there is no categorical pathway, so the child is not eligible."
+us,scenario_032,child1_head_start_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 130–150% FPL income limit, but Head Start's limit is 100% FPL. It also understated the family-of-three guideline at about $22,380 instead of $27,320. Measured against the correct $27,320 threshold, AGI of $30,916 is over the limit, so the child is not eligible."
+us,scenario_032,child1_head_start_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that income is 'within Head Start eligibility limits' without doing any comparison. AGI of $30,916, which includes $1,916 of taxable SS dependent benefits on top of $29,000 in wages, is above the 100% FPL guideline of $27,320, and no categorical pathway applies."
us,scenario_032,child1_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the 138% FPL adult Medicaid expansion cutoff to a six-year-old and then assigned the child to CHIP. Minnesota instead tests the child under the OLDER_CHILD Medicaid category, whose income limit includes MAGI at 1.56 times FPL."
us,scenario_032,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit eligibility indicator as grounds for denial instead of deriving eligibility from the supplied age, state, year, and household income. Minnesota's OLDER_CHILD Medicaid test covers this six-year-old at 1.56 times FPL."
us,scenario_032,child1_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly placed the child in a separate CHIP income band rather than applying Minnesota's OLDER_CHILD Medicaid threshold first. The child's MAGI of 1.56 times FPL satisfies the Medicaid limit for that age category.
@@ -2833,42 +2922,43 @@ us,scenario_032,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibilit
us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, substituted a roughly $16,150 MFJ standard deduction and a $1,000 CTC, and thereby produced $1,477 of pre-credit tax minus $1,000. Under the applicable 2026 parameters, the $2,200 CTC supplies $500 of nonrefundable credit, which fully reduces the liability to $0; its remaining $1,700 and the EITC are refundable and do not enter this output."
us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits."
us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits."
-us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly identified the 2026 one-child maximum EITC of $4,427 and placed $29,000 below the MFJ phase-out start. It then rounded the credit down to an 'approximately' $4,420 instead of using the exact $4,427, so its total of $4,420 + $1,700 = $6,120 is $7 short."
-us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It counted the 18-year-old spouse as a qualifying dependent and referred to '3 qualifying children', but a spouse on a joint return is never a qualifying child, so only the 6-year-old counts. It also used an invented $3,600 EITC and a $1,600 refundable CTC, and it applied a phase-out at $29,000 that does not reach this household, which sits below the MFJ one-child phase-out start. The correct figures are the full $4,427 EITC plus the $1,700 ACTC."
-us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It stated that the 2026 one-child maximum is $4,427 and that $29,000 is on the plateau. It then plugged in the 2025 maximum of $4,328 and added an arbitrary $215 'estimate refinement', producing $6,243 instead of $4,427 + $1,700 = $6,127."
-us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It phased the EITC down to $3,372 at $29,000 of earnings. The 2026 MFJ one-child phase-out starts above $31,000, above both earned income ($29,000) and AGI ($30,916), so the full $4,427 applies. Its reduction matches a single-filer threshold."
-us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It put $29,000 of MFJ earnings in the EITC phase-out range and cut the credit to $3,412 instead of the full $4,427. It then added a $1,100 'remaining' CTC amount on top of the $1,700 ACTC, but refundability is capped at $1,700 per child and the non-refundable remainder of the $2,200 CTC is unused at zero tax."
-us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a stale $27,638 MFJ phase-out start inflated to $28,329, which trimmed the EITC to $4,329. The actual 2026 MFJ start is above $31,000, so the full $4,427 applies. It also assumed a $2,000 CTC with a $30,000 standard deduction and computed taxable Social Security as $2,557 instead of $1,916, which created phantom tax. Its ACTC of $1,844 ignores the $1,700 per-child refundable cap; the correct 2026 standard deduction of $32,200 gives zero tax and exactly $1,700 of ACTC."
-us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It reached about $6,000 from the EITC plus the $1,700 ACTC. It then inflated the total to $8,200 by inventing an 'additional refundable credit for spouse as a qualifying dependent'. A spouse on a joint return cannot be a dependent, and no such credit exists. Its EITC maximum of $4,200-4,300 is also below the 2026 figure of $4,427."
-us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a one-child EITC maximum of $5,177 instead of the 2026 figure of $4,427. It also used a $27,460 MFJ phase-out threshold, well below the 2026 start of above $31,000. Its phased-out $4,625 still exceeds the true maximum, so the total overshoots at $6,325."
-us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the post-TCJA-sunset refundable CTC: 15% over $3,000, capped at $1,000. OBBBA made the $2,200 CTC permanent with a 2026 refundable cap of $1,700 and a $2,500 earnings floor, so the refundable CTC is $1,700. Its EITC of $4,488 also overstates the 2026 one-child maximum of $4,427."
-us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It phased the EITC down to $3,797 even though earned income and AGI are below the 2026 MFJ one-child phase-out start, so the full $4,427 applies. It used the expired $1,000 refundable CTC instead of the $1,700 OBBBA cap. Its reasoning totals $4,797 but it submitted $5,430, and neither matches $6,127."
-us,scenario_032,federal_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It started the EITC phase-out at $23,052, a single-filer-level threshold that omits the MFJ increase. It also understated the maximum at $4,297, which drove the EITC down to $3,040. The 2026 MFJ one-child phase-out starts above $31,000, so the full $4,427 applies; its $1,700 ACTC was correct."
-us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the expired post-TCJA $1,000 refundable CTC instead of the 2026 OBBBA refundable cap of $1,700. It also reduced the EITC to $3,599.34 even though the household is below the MFJ one-child phase-out start and receives the full $4,427."
-us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no breakdown. Its $5,462 fits an EITC near the maximum plus the expired $1,000 refundable CTC, not the 2026 $1,700 ACTC cap. The correct total is the full $4,427 EITC plus the $1,700 ACTC."
-us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly applied the 'pre-TCJA' $1,000 Additional Child Tax Credit, but OBBBA made the $2,200 CTC permanent with a 2026 refundable cap of $1,700. Its $4,440 EITC also overstates the 2026 one-child maximum of $4,427."
-us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $1,000 ACTC, the expired post-sunset cap, instead of the 2026 $1,700 refundable cap. It also set the EITC at $4,014 even though the household is on the plateau below the MFJ phase-out start and receives the full $4,427."
-us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no breakdown. The correct figure is the full $4,427 EITC plus the $1,700 ACTC, because earned income of $29,000 and AGI of $30,916 are below the 2026 MFJ one-child phase-out start and tax is zero. Its $3,745 is $2,382 short, which requires phasing out an EITC that is not in phase-out and/or using the expired $1,000 ACTC."
-us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used a $1,000 ACTC with a $3,000 floor, but OBBBA made the $2,200 CTC permanent with a $1,700 refundable cap and a $2,500 floor. Its EITC of $4,428 is within $1 of the correct figure, so the $700 ACTC shortfall drives the error. It also misattributed the head's Social Security benefits to the child."
-us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $5,248 is $879 short of the full $4,427 EITC plus the $1,700 ACTC. That gap fits the expired $1,000 refundable CTC paired with a slightly low EITC, instead of the 2026 $1,700 cap and the full one-child maximum."
-us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It gave no breakdown. Its $4,056 is $2,071 short of the full $4,427 EITC plus the $1,700 ACTC. It either phased out an EITC that sits on the MFJ plateau or understated the refundable CTC below the $1,700 cap."
-us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so no answer was scored."
-us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the entire $2,200 2026 CTC as refundable. The refundable portion is capped at $1,700 per child, so the ACTC is $1,700. It also used an EITC maximum of $4,095 instead of the 2026 one-child maximum of $4,427."
-us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no refundable credit. The household has $29,000 of earned income and a qualifying 6-year-old, so it qualifies for the full $4,427 EITC and the $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed refundable credits were zero because no credit amounts were 'specified' in the facts. It never computed the EITC ($4,427 for one child, MFJ, on the plateau) or the $1,700 refundable CTC that the household's $29,000 of earnings and qualifying child generate."
-us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its $4,427 EITC was correct, but it used a refundable CTC limit of $1,800. The 2026 per-child refundable cap is $1,700, so it overstated the total by $100."
-us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It named the correct two credits but gave no amounts. Its $6,047 is $80 below $4,427 + $1,700, which implies an EITC of about $4,347 instead of the 2026 one-child maximum of $4,427 that applies on the MFJ plateau."
-us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,other,False,"It gave only a one-line justification with no computation. Its $2,500 falls far below the full $4,427 EITC plus the $1,700 refundable CTC, so it did not carry out either credit's calculation."
-us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the post-sunset $1,000 CTC (15% over $3,000, capped at $1,000) instead of OBBBA's $2,200 CTC with a 2026 refundable cap of $1,700. Its projected EITC of $4,436 also overstates the 2026 one-child maximum of $4,427."
-us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a 'post-sunset' additional CTC of $1,000, but OBBBA sets the 2026 refundable cap at $1,700. It also set the one-child EITC maximum at $4,318 instead of $4,427."
-us,scenario_032,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $1,700 ACTC and plateau placement were right, but it projected the 2026 one-child EITC maximum as $4,436 instead of the actual $4,427, overshooting by $9."
-us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It treated the CTC as $1,000 and fully refundable, but the 2026 CTC is $2,200 with a $1,700 refundable cap, which this household reaches because tax is zero and 15% of $26,500 is $3,975. It also projected the EITC maximum as $4,439 instead of $4,427."
-us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its $1,700 ACTC and plateau placement were right, but it projected the 2026 one-child EITC maximum as $4,449 instead of $4,427, overshooting by $22."
-us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so no answer was scored."
-us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"It computed the EITC for two children even though only the 6-year-old qualifies, because the spouse cannot be a qualifying child. It then left the refundable CTC out of the total entirely and reported only the $4,205 EITC. The correct total is the one-child EITC of $4,427 plus the $1,700 ACTC."
-us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It began the EITC phase-out at about $22,190, a single-filer-level threshold, and cut the credit to $3,348. The 2026 MFJ one-child phase-out starts above $31,000, so the full $4,427 applies; its $1,700 ACTC was correct."
-us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It invented one-child EITC maximums of $5,386 to $5,640 and phase-out starts near $22,000, which yielded an EITC of $4,509 instead of the full $4,427. It then arbitrarily 'limited' the refundable CTC to $800, even though tax is zero and 15% of $26,500 exceeds the $1,700 cap, so the ACTC is $1,700."
-us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It reported an EITC of $1,171 and an ACTC of $937, both badly understated. The EITC reflects a deep phase-out that does not apply: earned income of $29,000 and AGI of $30,916 are below the MFJ one-child phase-out start, so the full $4,427 applies. The ACTC ignores that 15% of ($29,000 - $2,500) = $3,975 hits the $1,700 cap."
+us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The structure was right: full one-child EITC because AGI is below the MFJ phaseout, plus a $1,700 ACTC. But the model rounded the EITC down to ~$4,420 instead of using the published 2026 maximum of $4,427, which leaves the total $7 short."
+us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted the 18-year-old spouse as a qualifying dependent and applied a 3-child EITC schedule, even though only the 6-year-old qualifies. It also used a $1,600 refundable CTC instead of $1,700 and applied an arbitrary phaseout. The correct amount is the full $4,427 one-child EITC plus a $1,700 ACTC."
+us,scenario_032,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model claimed no refundable credit applies. In fact, an MFJ couple with $29,000 of wages and a 6-year-old qualifying child gets the full $4,427 one-child EITC and a $1,700 refundable ACTC."
+us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025-level EITC of ~$4,328 instead of the 2026 maximum of $4,427, which AGI of $30,916 fully qualifies for. It then added an unexplained $215 'estimate refinement', which pushed the total above $6,127."
+us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model placed $29,000 of earned income inside the one-child EITC phaseout and cut the credit to $3,372. The 2026 MFJ phaseout does not start until about $31,160, which is above both earnings and AGI of $30,916, so the full $4,427 applies."
+us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model phased the EITC down to $3,412 even though income is below the MFJ phaseout start, so the full $4,427 applies. It then added a fabricated '$1,100 remaining' credit on top of the $1,700 ACTC."
+us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $2,000 CTC with no $1,700 refundable cap, so its ACTC came out at $1,844. That figure came from a positive tax liability, which was wrong for two reasons: it used the outdated $30,000 standard deduction instead of $32,200, and it used the 85% tier for taxable Social Security instead of the 50% tier (taxable SS is $1,916). It also started the one-child EITC phaseout at ~$28,329 instead of ~$31,160, which cut the EITC to $4,329 instead of the full $4,427."
+us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly reaching about $4,300 + $1,700, the model added an invented 'additional refundable credit for spouse as a qualifying dependent'. A spouse on a joint return is never a dependent and generates no refundable credit, so the answer should have been the $4,427 EITC plus the $1,700 ACTC."
+us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a nonexistent $5,177 one-child EITC maximum and an outdated $27,460 MFJ phaseout threshold. With the 2026 parameters ($4,427 maximum, phaseout starting around $31,160), AGI of $30,916 gets the unreduced $4,427, not $4,625."
+us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the expired pre-TCJA ACTC rules ($1,000 cap, $3,000 earnings threshold). OBBBA set the 2026 CTC at $2,200 with a $1,700 refundable cap. It also overstated the one-child EITC as $4,488 instead of $4,427."
+us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the sunset-era $1,000 refundable CTC instead of the $1,700 OBBBA cap. It also phased the EITC down to $3,797 even though AGI is below the MFJ phaseout start, so the full $4,427 applies. Its stated total ($4,797) does not even match its submitted $5,430."
+us,scenario_032,federal_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model started the one-child EITC phaseout at ~$23,052, which is roughly the single-filer threshold, and omitted the MFJ add-on, so the EITC was cut to $3,040. With the 2026 MFJ threshold of about $31,160, AGI of $30,916 gets the full $4,427; the $1,700 ACTC was correct."
+us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model phased the EITC down to $3,599 even though AGI is below the MFJ phaseout threshold, so the full $4,427 applies. It also used the expired pre-TCJA $1,000 ACTC instead of the $1,700 2026 refundable cap."
+us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The $5,462 total implies an EITC of about $4,462 plus the expired pre-TCJA $1,000 ACTC. Under OBBBA, the 2026 refundable CTC cap is $1,700 and the one-child EITC maximum is $4,427."
+us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly used the 'pre-TCJA Additional Child Tax Credit of $1,000', assuming TCJA had sunset. OBBBA set the 2026 CTC at $2,200 with a $1,700 refundable portion. It also used $4,440 instead of the $4,427 EITC maximum."
+us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the expired $1,000 ACTC instead of the $1,700 2026 refundable cap. It also understated the EITC at $4,014, although AGI below the MFJ phaseout start qualifies for the full $4,427."
+us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"$3,745 is less than the EITC alone. The correct total is the full $4,427 one-child EITC (AGI of $30,916 is below the MFJ phaseout start) plus the $1,700 ACTC. The model's number reflects a phased-down EITC and a missing or understated refundable CTC."
+us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and applied a $1,000 CTC with a $3,000 earnings threshold. OBBBA made the CTC $2,200 per child with a $1,700 refundable cap, so the ACTC is $1,700, not $1,000. Its EITC of $4,428 was essentially correct."
+us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $5,248 total is consistent with an EITC of about $4,248 plus the expired pre-TCJA $1,000 ACTC. The correct components are the full $4,427 EITC and the $1,700 OBBBA refundable CTC cap."
+us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $4,056 total is $2,071 short of the correct $4,427 EITC + $1,700 ACTC. It is consistent with a phased-down EITC paired with the expired $1,000 ACTC, even though AGI of $30,916 is below the MFJ phaseout start."
+us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no usable value for federal_refundable_credits. The correct derivation is the $4,427 one-child EITC plus the $1,700 refundable ACTC, for $6,127."
+us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 CTC as refundable and ignored the $1,700 refundable (ACTC) cap. It also understated the one-child EITC maximum as $4,095 instead of $4,427."
+us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable credit. A working MFJ couple with $29,000 of wages and a qualifying 6-year-old gets the $4,427 EITC and the $1,700 refundable ACTC."
+us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed zero refundable credits because no credit amounts were listed in the inputs. Credits are computed, not given: the 6-year-old qualifies the couple for the $4,427 one-child EITC and the $1,700 ACTC."
+us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The EITC of $4,427 was correct, but the model used an $1,800 refundable CTC limit instead of the 2026 $1,700 cap, which overstated the total by $100."
+us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the right two credits but submitted $6,047, $80 below $4,427 + $1,700. The number is consistent with an EITC near $4,347 instead of the full 2026 one-child maximum of $4,427."
+us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave a rough $2,500 guess with no computation. The full one-child EITC is $4,427 because AGI of $30,916 is below the MFJ phaseout start, and the ACTC is $1,700, so the guess misses about $3,600."
+us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the sunset-era $1,000 CTC with a $3,000 earnings threshold instead of OBBBA's $2,200 CTC with a $1,700 refundable cap. It also projected the EITC at $4,436 instead of the published $4,427."
+us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the 'post-sunset' $1,000 ACTC, but OBBBA kept the higher CTC, so the 2026 refundable portion is $1,700. It also understated the one-child EITC maximum as $4,318 instead of $4,427."
+us,scenario_032,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The structure was correct: full EITC plus a $1,700 ACTC. But the model used a projected one-child EITC maximum of $4,436 instead of the 2026 parameter of $4,427, which overstated the total by $9."
+us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model treated the CTC as $1,000 (pre-TCJA) instead of the 2026 $2,200 credit with a $1,700 refundable cap. It also overstated the EITC as $4,439 instead of $4,427."
+us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly found the full EITC plus a $1,700 ACTC, but used an EITC maximum of $4,449 instead of the 2026 one-child maximum of $4,427, which overstated the total by $22."
+us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value for federal_refundable_credits. The correct derivation is the $4,427 one-child EITC plus the $1,700 refundable ACTC, for $6,127."
+us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model applied a two-child EITC schedule even though only the 6-year-old is a qualifying child. It then left the refundable CTC out of the total entirely, reporting the EITC alone instead of $4,427 + $1,700."
+us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used a ~$22,190 phaseout threshold, roughly the single-filer level, and omitted the MFJ add-on, so the EITC was cut to $3,348. The 2026 MFJ threshold is about $31,160, above AGI of $30,916, so the full $4,427 applies; its $1,700 ACTC was correct."
+us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model cycled through invented EITC maximums ($5,640, $5,386) and single-level phaseout thresholds before settling on $4,509. It then arbitrarily limited the ACTC to $800, even though 15% of $26,500 is $3,975 and well above the $1,700 cap. The correct amounts are a $4,427 EITC and a $1,700 ACTC."
+us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model phased the EITC down to $1,171 and computed a $937 ACTC. AGI of $30,916 is below the MFJ phaseout start, so the full $4,427 applies, and 15% of earnings over $2,500 ($3,975) exceeds the $1,700 cap, so the ACTC is the full $1,700."
us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Compared $42,664 with 130% FPL (~$34,645) and ruled out SNAP/TANF categorical eligibility, but never applied Minnesota's universal free school meals program. That program makes the 6-year-old K-12 child eligible for free meals whatever the household earns."
us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Treated the 130% FPL income test and TANF/SNAP direct certification as the only free-meal pathways and overlooked Minnesota's universal free school meals program, which gives the K-12 child FREE tier status. Its ~170% FPL figure also overstates the engine's 1.56 ratio, but the missed state program is what decides the answer."
us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,Applied only the federal 130% FPL free-meal limit to income at ~160% FPL and ignored that Minnesota provides universal free school meals. Under that program the household's K-12 child qualifies for free meals whatever the income.
@@ -2911,46 +3001,47 @@ us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_dedu
us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed $42,664 in MAGI, then dropped the Social Security dependent benefits on the theory that a child receives them. It tested only the $29,000 in wages. The benefits are the Head's own income, and Medicaid MAGI counts them in full, so the Head is at 156% FPL, above the 138% limit."
us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either."
-us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It answered federal payroll tax alone, $1,798 Social Security plus $420.50 Medicare = $2,218.50, saying Minnesota has no mandatory employee state payroll tax in 2026. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 for $2,218.50, federal payroll tax alone, saying there is no modeled mandatory Minnesota employee payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no mandatory state payroll tax on employees and reached $2,218.50, then cut that to $2,175.90 for unexplained 'state-specific considerations', with no rule behind the change. The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It computed 7.65% × $29,000 = $2,218.50, federal payroll tax alone, saying Minnesota has no mandatory employee state payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It got Social Security ($1,798) and Medicare ($420.50) right and stopped at $2,218.50, saying Minnesota has no mandatory employee payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It stopped at federal FICA of $2,218.50, saying there is no mandatory Minnesota employee payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 for $2,218.50, federal payroll tax alone, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota imposes no employee payroll tax beyond the federal ones and answered $2,218.50. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no state disability or paid family leave employee contribution in 2026 and answered $2,218.50. Minnesota Paid Leave premiums do start in 2026: the reference adds $127.60 (0.44% of $29,000), the most the employer may deduct. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It assumed no mandatory Minnesota employee state payroll tax is modeled and stopped at $2,218.50. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 for $2,218.50 and said there is no employee state payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It counted a Minnesota Paid Leave employee share, but at 0.35% ($101.50), for $2,320. The reference counts the most the employer may deduct, 0.44% of the $29,000 in paid-leave taxable wages, or $127.60, so the answer is $26.10 under it; without the share, which the law lets the employer pay itself, the total is $2,218.50."
-us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Its stored explanation says no employee state payroll tax applies and totals $2,218.50, federal tax alone. Its recorded value, $2,320, matches another response in its raw output, which added a $101.50 Minnesota Paid Leave premium (0.35% of $29,000). The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It computed only the 7.65% employee FICA, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare on $29,000, rounded to $2,219, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 for $2,218.50 and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It computed only federal FICA at 7.65%, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare on $29,000, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It treated payroll tax as 7.65% of wages, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It computed 7.65% employee FICA only, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It got Social Security ($1,798) and Medicare ($420.50) right and stopped at $2,218.50, without mentioning Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no payroll_tax value and no explanation, so there was no answer to score. The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 for $2,218.50 and ruled out Additional Medicare Tax, without mentioning Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Its explanation lists only 6.2% Social Security and 1.45% Medicare on $29,000 and says Minnesota has no mandatory state employee payroll tax, which would total $2,218.50. The $4,428.60 it submitted is about double that, close to the 15.3% employer-plus-employee FICA total ($4,437). The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It first computed $2,218.50, then threw it out for an invented 5.24% rate ($1,520) tied to a Social Security wage-base cap that does not bind at $29,000. It treated Minnesota's employee payroll tax as unspecified and added none. The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It stopped at FICA of $2,218.50, saying there is no additional or state payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $29,000, rounded to $2,219, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It answered 7.65% of $29,000, $2,218.50, saying no other employee-side payroll tax is indicated. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It computed only 7.65% FICA on $29,000, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied only Social Security and Medicare to $29,000, $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It stopped at $2,218.50, saying no mandatory Minnesota employee payroll tax applies. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It stopped at FICA of $2,218.50, saying there is no Minnesota employee-side payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"It answered federal payroll tax alone, $2,218.50, saying Minnesota paid-leave withholding is optional. The law does leave the deduction to the employer, which owes the premium and may deduct up to 0.44% of wages from the employee or pay that share itself. The reference counts the most the employer may deduct, $127.60 (0.44% of $29,000), for $2,346.10; the prompt does not say what this employer does."
-us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It summed only Social Security ($1,798) and Medicare ($420.50), $2,218.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It answered $2,218.50, saying no state employee payroll taxes apply. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no payroll_tax value and no explanation, so there was no answer to score. The reference is $1,798 + $420.50 + $127.60 = $2,346.10, counting the most the employer may deduct for Minnesota Paid Leave (0.44% of $29,000); the law lets the employer pay that share itself, and without it the total is federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50, rounded to $2,219, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It stopped at $2,218.50, saying Minnesota levies no mandatory employee payroll tax. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no mandatory employee state payroll tax because it has no CA/NY-style disability insurance tax, and rounded $2,218.50 to $2,219, without mentioning Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
-us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA, $2,218.50, miswriting Medicare as $419.50 instead of $420.50, and did not mention Minnesota Paid Leave. The reference adds $127.60 (0.44% of $29,000), the most the employer may deduct for Minnesota Paid Leave. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone, $2,218.50."
+us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly ($1,798 + $420.50) but said Minnesota has no employee payroll tax in 2026, so it left out the $127.60 Paid Leave contribution (0.44% of $29,000). Its $2,218.50 fits the reading where the employer pays the whole 0.88% premium. The reference instead assumes the employer withholds the full 0.44% employee share."
+us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $29,000 and treated Minnesota as having no mandatory employee payroll tax, so it left out the 0.44% Paid Leave employee share ($127.60). That fits the reading where the employer pays the full premium, not the reference's reading where the employer withholds the maximum employee share."
+us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It derived $2,218.50 in FICA several times, then reported $2,175.90 after an unexplained 'state-specific adjustment' that matches no rule. It also never added the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). The correct total is $2,346.10."
+us,scenario_032,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It rounded 7.65% FICA on $29,000 to $2,219 and said Minnesota has no mandatory employee payroll tax, so it left out the $127.60 Paid Leave contribution (0.44% employee share). That follows the reading where the employer pays the full premium. The reference assumes the employee share is withheld."
+us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA on $29,000 and said Minnesota has no mandatory employee payroll tax, which drops the 0.44% Paid Leave employee contribution ($127.60). Its $2,218.50 matches the employer-pays-all reading, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($1,798) and Medicare ($420.50) correctly but said Minnesota has no mandatory employee payroll tax, so it left out the $127.60 Paid Leave employee share (0.44% of wages). That is the employer-pays-full-premium reading. The reference assumes the employer withholds the employee half."
+us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It counted only employee FICA ($2,218.50) and said Minnesota has no mandatory employee payroll tax, leaving out the 0.44% Paid Leave contribution of $127.60. That fits the reading where the employer pays the whole premium, not the reference's assumption that the employee share is withheld."
+us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% FICA rate to $29,000 and never considered the Minnesota Paid Leave employee contribution (0.44%, $127.60). Its total matches the reading where the employer pays the full premium, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax beyond federal FICA, missing that Minnesota Paid Leave (effective 2026) lets the employer withhold a 0.44% employee share, which is $127.60 here. Its $2,218.50 matches the employer-pays-all reading. The reference assumes the share is withheld."
+us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no paid family leave employee contribution in 2026. In fact Minnesota Paid Leave began January 1, 2026, and the engine withholds a 0.44% employee share ($127.60). Its FICA-only $2,218.50 fits the reading where the employer pays the whole premium."
+us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It assumed no Minnesota employee payroll tax is modeled, so it left out the 0.44% Paid Leave contribution on $29,000 ($127.60). That is the employer-pays-full-premium reading, not the reference's maximum-employee-withholding reading."
+us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA ($2,218.50) and said no employee state payroll tax applies, so it left out the $127.60 Minnesota Paid Leave employee share (0.44%). That follows the reading where the employer pays the full premium, not the reference's withholding assumption."
+us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It correctly included Minnesota Paid Leave but used a 0.35% employee rate ($101.50) instead of 0.44%, which is half of the 0.88% total premium ($127.60). That understated the total by $26.10: it answered $2,320 instead of $2,346.10."
+us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Its submitted $2,320 contradicts its own explanation, which says no state payroll tax and gives $2,218.50. The $2,320 equals FICA plus a 0.35% Paid Leave rate, not the correct 0.44% employee share ($127.60), which gives $2,346.10."
+us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA to $29,000 and left out the Minnesota Paid Leave employee contribution (0.44%, $127.60). That fits the reading where the employer pays the full premium. The reference assumes the maximum employee share is withheld."
+us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It computed employee FICA only, rounded to $2,219, and never added the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). That follows the employer-pays-all reading, not the reference's withholding reading."
+us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It calculated 29,000 × 7.65% = $2,218.50 and left out the 0.44% Minnesota Paid Leave employee share ($127.60). Its answer fits the reading where the employer pays the whole premium, not the reference's assumption of maximum employee withholding."
+us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted only the 7.65% FICA share and left out the Minnesota Paid Leave employee contribution of 0.44% on $29,000 ($127.60). That matches the employer-pays-full-premium reading. The reference assumes the employee share is withheld."
+us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It reported FICA only ($2,218.50) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That fits the reading where the employer pays the full premium, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 and did not add the 0.44% Minnesota Paid Leave employee contribution ($127.60). That is the employer-pays-all reading, not the reference's assumption that the employer withholds the maximum employee share."
+us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It computed employee FICA of $2,218.50 and left out the $127.60 Minnesota Paid Leave employee share (0.44% of wages). That follows the reading where the employer pays the full 0.88% premium."
+us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It summed Social Security ($1,798) and Medicare ($420.50) but left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). That matches the employer-pays-all reading, not the reference's withholding reading."
+us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee share = $2,346.10."
+us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $29,000 and left out the Minnesota Paid Leave employee contribution (0.44%, $127.60). That follows the reading where the employer pays the full premium, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Its explanation describes only the employee 6.2% + 1.45% rates, but it submitted $4,428.60. That is roughly double the $2,218.50 employee FICA, which suggests it added the employer FICA match that the question excludes. It also left out the $127.60 Minnesota Paid Leave employee share; the correct total is $2,346.10."
+us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It correctly computed 7.65% × $29,000 = $2,218.50, then dropped that for an invented 5.24% rate tied to a nonexistent wage-base interaction ($1,520). $29,000 is far below the Social Security wage base, so the full 6.2% applies. It also never added the $127.60 Minnesota Paid Leave contribution."
+us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50 and said no state payroll tax applies, leaving out the 0.44% Minnesota Paid Leave employee share ($127.60). That is the employer-pays-full-premium reading, not the reference's withholding reading."
+us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA (rounded to $2,219) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). That fits the reading where the employer pays the whole premium."
+us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It counted only 7.65% FICA and said no other employee payroll tax was indicated, which drops the 0.44% Minnesota Paid Leave contribution ($127.60). That matches the employer-pays-all reading, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It reported 7.65% of $29,000 and left out the Minnesota Paid Leave employee share (0.44%, $127.60). That follows the reading where the employer pays the full premium, not the reference's withholding assumption."
+us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It computed only Social Security and Medicare on $29,000 and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That fits the reading where the employer pays the whole premium."
+us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It said no mandatory Minnesota employee payroll tax applies, which drops the 0.44% Paid Leave employee share ($127.60) the engine withholds. Its $2,218.50 matches the employer-pays-full-premium reading."
+us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax and so left out the 0.44% Paid Leave contribution on $29,000 ($127.60). That is the employer-pays-all reading. The reference assumes the employer withholds the maximum employee share."
+us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly saw that Minnesota Paid Leave employee withholding is optional for the employer and excluded it, giving $2,218.50. The reference takes the other reading: the employer withholds the maximum 0.44% employee share ($127.60), for $2,346.10."
+us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It summed $1,798 Social Security and $420.50 Medicare and never considered the 0.44% Minnesota Paid Leave employee contribution ($127.60). That fits the reading where the employer pays the full premium."
+us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It said no state employee payroll taxes apply, leaving out the $127.60 Minnesota Paid Leave employee share (0.44% of $29,000). That follows the employer-pays-all reading, not the reference's maximum-withholding reading."
+us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation. The correct total is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee contribution = $2,346.10."
+us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It rounded employee FICA to $2,219 and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). That matches the reading where the employer pays the full premium, not the reference's withholding reading."
+us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It said Minnesota levies no mandatory employee payroll tax, which drops the 0.44% Paid Leave employee share ($127.60) that started in 2026. Its $2,218.50 fits the employer-pays-full-premium reading."
+us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It dismissed Minnesota payroll taxes because Minnesota has no disability insurance tax like CA or NY, overlooking the 2026 Minnesota Paid Leave employee share (0.44%, $127.60). Its rounded $2,219 follows the reading where the employer pays the full premium."
+us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It reported 7.65% FICA ($2,218.50, though it misstated Medicare as $419.50) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That matches the employer-pays-all reading, not the reference's maximum-withholding reading."
us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly placed $42,664 between 130% and 185% FPL for a 3-person household. It then applied the federal reduced-price band alone and ignored Minnesota's universal free school meals program. That program puts every student in the FREE tier, which replaces reduced-price eligibility."
us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted a 4-person household with two children, but the household has 3 people and one child. Its answer rests on the federal 185% FPL reduced-price limit alone. It missed that Minnesota's universal free school meals program gives the child the FREE tier, so reduced-price support is not returned."
us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that income above 130% FPL rules out free meals, which leaves only reduced-price. That reasoning ignores Minnesota's universal free school meals program, which gives free meals to every student regardless of income. The household is therefore in the FREE tier, not the reduced-price tier."
@@ -3018,273 +3109,283 @@ us,scenario_032,state_income_tax_before_refundable_credits,glm-5.3,llm_error,sta
us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted that Minnesota AGI equals federal AGI ""with no modifications,"" discarding the Minnesota Social Security subtraction that removes the entire $1,916 of federally taxable benefits at this income, and it also omitted the Minnesota dependent exemption of roughly $5,300 for Child 1. Applying that subtraction against its own $30,699 standard deduction gives $29,000 − $30,699 < 0, so taxable income is zero and the tax is $0, not the $217 base it taxed at 5.35%."
us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs object. This is a failure to deliver the answer under the output contract, not a substantive error in the Minnesota computation."
us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model declared the full $13,664 of Social Security ""fully taxable"" for Minnesota purposes, contradicting the §86 provisional-income formula (provisional income $35,832 against the $32,000 joint base makes only $1,916 taxable) and the Minnesota Social Security subtraction that removes even that $1,916, and it used a $28,500 joint standard deduction below Minnesota's 2026 amount with no dependent exemption. Its own stated chain of $42,664 − $28,500 at 5.35% yields $758, not the $128 it submitted, so the figure is an unsupported approximation layered on an inflated income base; the correct $29,000 base falls below the deduction and yields $0."
-us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model added a working-family amount of about $925 for a filer with one child. Under the CWFC, the working-family part does not grow with children: it is 4% of earnings capped at about $365. Its $2,337 total is about $150 above the correct $2,187.60, which is the indexed child credit plus the capped working-family part."
-us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model added a one-child add-on of about $950 to the working-family part. The 2023 CWFC removed any per-child add-on, so the working-family part is only 4% of earnings capped at about $365. The model's $3,100 overstates the correct $2,187.60 by roughly $900."
-us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said Minnesota has no refundable individual income tax credits. The Child and Working Families Credit is fully refundable, and this household qualifies through one child aged 6 and $29,000 of earnings, for $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used the unindexed 2023 child credit of $1,750 and a working-family part of about $410. It also gave the working-family part its own phaseout starting near $33,000. The correct 2026 figures are the indexed child credit plus a working-family part capped at about $365, which together equal $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model counted only the $1,750 child credit and dismissed the working-family part as small. It left out the 4%-of-earnings part (capped at about $365) and did not inflation-index the child credit, so it fell short of $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model put the phaseout threshold near $31,000 and reduced the credit through a shared phaseout. The 2026 joint-filer threshold is far above the $30,916 AGI, so no reduction applies. It also used the unindexed $1,750 child credit, and its offsetting approximations produced $2,185 instead of the full indexed total of $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model had the right structure and no phaseout, but used the 2023 base amounts ($1,750 child credit and $350 working-family cap). Minnesota indexes both amounts for inflation, which brings the 2026 total to $2,187.60 rather than $2,100."
-us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model treated the Working Family Credit as 35% of the federal EITC, which is the pre-2023 structure. It also left out the per-child credit entirely and included 85% of the Social Security benefits in AGI. Under the CWFC, the answer is the indexed child credit plus 4% of earnings capped at about $365, for $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model concluded that no refundable credit applies without rent, education, or other expenses. The CWFC requires only earnings and a qualifying child. The 6-year-old child and $29,000 of wages produce $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model estimated a working-family part of about $1,900 for one child using the repealed EITC-linked schedule. Under the CWFC, that part is only 4% of earnings capped at about $365, so the total is $2,187.60, not $3,700."
-us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 45% of a $4,625 federal EITC, a method Minnesota repealed in 2023. It then added that $2,081 to the child credit. The actual working-family part is 4% of earnings capped at about $365, so the correct total is $2,187.60."
-us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model calculated the working-family part as 33% of the federal EITC ($1,481), which is the old structure. The actual part is 4% of earnings capped at about $365. It also overstated the indexed child credit at $1,912, so its $3,393 exceeds $2,187.60."
-us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model gave a $1,633 working-family credit and a $1,950 child credit. The working-family part is capped at about $365 (4% of earnings), and the indexed child credit is about $1,823, so the correct total is $2,187.60, not $3,583."
-us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 45% of the federal EITC ($1,368), which is the repealed structure. Under the CWFC, the working-family part is 4% of earnings capped at about $365, and it combines with the indexed child credit for $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model cut the child credit to $983.60 even though it said the phaseout starts at $35,000, above the $30,916 AGI. In fact no phaseout applies. The model also overstated the working-family part at $550, when the cap is about $365. The full credit is $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said no Minnesota refundable credits apply. The household qualifies for the fully refundable CWFC through the 6-year-old child and $29,000 of earnings, which yields $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model gave a $700 working-family amount to the couple, effectively $350 per spouse. The working-family part is one amount per return: 4% of earnings capped at about $365. With the unindexed $1,750 child credit on top, the model reached $2,450 instead of $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model assigned about $1,200 to the Working Family Credit using the old EITC-style schedule. Under the CWFC, the working-family part is 4% of earnings capped at about $365, so the total is $2,187.60, not $2,950."
-us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave $600 without any breakdown. That figure fits a working-family-sized amount with no child credit. The per-child credit of about $1,823 (indexed) is missing, and it makes up most of the $2,187.60 CWFC."
-us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model counted only the unindexed $1,750 child credit. It left out the working-family part (4% of earnings capped at about $365) and the 2026 inflation indexing that brings the total to $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model added a working-family amount of about $1,085 to the $1,750 child credit, using the repealed EITC-linked schedule. The CWFC working-family part is capped at about $365, so the correct total is $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"The model's $2,450 matches $1,750 plus a doubled $700 working-family amount, one $350 cap per spouse. The working-family part is a single per-return amount capped at about $365, and the child credit is indexed, so the total is $2,187.60."
-us,scenario_032,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model stated that no other refundable credits apply and counted only an indexed child credit of $1,912. It left out the CWFC working-family part (4% of $29,000 of earnings, capped at about $365) and overstated the indexed child amount, which is about $1,823. The combined credit is $2,187.60."
-us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model phased down the child credit from a $29,500 threshold, which is the 2023 threshold for non-joint filers. This household files jointly, and the 2026 joint threshold is well above the $30,916 AGI, so the full credit of $2,187.60 applies. The model also understated the working-family part at $180."
-us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable Minnesota credit. The household qualifies for the refundable CWFC through the 6-year-old child and $29,000 of wages, for $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required explicitly stated credit amounts and so returned zero. The CWFC is calculated from the facts given (a qualifying child, earnings, and AGI below the phaseout threshold) and comes to $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the 2023 base amounts ($1,750 child credit and $350 working-family cap) with no inflation indexing. The indexed 2026 amounts total $2,187.60, not $2,100."
-us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model calculated the working-family part as 25% of a $4,427 federal EITC, a method that no longer exists. Under the CWFC, that part is 4% of earnings capped at about $365, so the total is $2,187.60, not $2,856.75."
-us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model assigned a $1,675 Working Family Credit using the pre-2023 schedule tied to the number of children. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model's $3,572 fits a child credit plus an old-style working-family amount of about $1,800 tied to the EITC. Under the CWFC, the working-family part is capped at about $365, so the total is $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the unindexed 2023 child credit of $1,750 and added a $388 working-family amount. The 2026 indexed child credit is about $1,823, and with the capped working-family part the total is $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model's $4,500 overstates the working-family part as a large EITC-style amount. The CWFC working-family part is capped at about $365 per return, and the only other part is the indexed child credit, for $2,187.60 in total."
-us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model assigned a $1,275 working-family credit using the repealed EITC-linked schedule. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model used the unindexed $1,750 child credit and a $387 working-family amount. The 2026 child credit is inflation-indexed to about $1,823, and the combined unphased credit is $2,187.60."
-us,scenario_032,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model counted only a $500 working-family amount and left out the refundable per-child credit, which is about $1,823 after indexing. The CWFC combines both parts for $2,187.60."
-us,scenario_032,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model calculated the working-family part as 40% of the federal EITC ($1,774), which is the repealed pre-2023 structure. The CWFC working-family part is 4% of earnings capped at about $365, so the total is $2,187.60."
-us,scenario_032,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model estimated a $1,560 Working Family Credit using the old schedule tied to the number of children. The CWFC working-family part is capped at about $365, which with the indexed child credit gives $2,187.60."
-us,scenario_032,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model applied a one-child maximum Working Family Credit of $1,577, which comes from the schedule repealed in 2023. The CWFC working-family part is a flat 4% of earnings capped at about $365 regardless of the number of children, so the total is $2,187.60."
-us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 33% of the federal EITC ($1,465), which is the repealed structure. The CWFC working-family part is 4% of earnings capped at about $365, and it combines with the indexed child credit for $2,187.60."
-us,scenario_032,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model estimated a one-child Working Family Credit of about $1,300 using the old schedule. The CWFC working-family part is capped at about $365, so the total is $2,187.60, not $3,050."
-us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so there is no answer to score."
-us,scenario_032,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used a full one-child Working Family Credit of $1,250 from the repealed schedule. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60."
-us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model ruled the household out because the spouse has no earnings and because it believed income exceeded the thresholds. The CWFC needs only one earner on the joint return, and the $30,916 AGI is below the joint phaseout threshold, so the full $2,187.60 is paid."
-us,scenario_032,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model assumed a Working Family Credit maximum of about $2,900 phased down to $1,650. The CWFC working-family part is capped at about $365, and no phaseout applies at this AGI, so the total is $2,187.60."
-us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model counted only an EITC-linked Working Family Credit of about $1,895 and left out the refundable per-child credit. The CWFC is the indexed child credit plus a working-family part capped at about $365, for $2,187.60."
-us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model claimed a $241 child and dependent care credit even though the household has no care expenses. It also used an old-style $1,648 Working Family Credit and left out the per-child credit. The correct CWFC total is $2,187.60."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $243 state tax refund from gross income, which gave AGI of $83,208 and taxable income of $35,708 instead of $35,951.23, and its own brackets then produced $3,789. It submitted $3,699 anyway, citing an unexplained 'PolicyEngine-style rounding' adjustment that nothing in its computation supports."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI ($83,451), standard deduction ($35,500) and senior deduction ($12,000) were all correct, but it also subtracted $1,225 of auto-loan interest as an OBBBA vehicle-loan deduction. The household facts do not establish a qualifying new vehicle loan, so that deduction does not apply. Taking it cut taxable income to $34,726 and tax by $147."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $11,400 Schedule C loss as a capital loss capped at $3,000, when the full loss offsets ordinary income. It subtracted the standard deduction while computing AGI, used a $30,400 standard deduction with no aged additions and no $12,000 senior deduction, and then submitted $5,661, which does not follow from its own $61,451 taxable income."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 state refund, which gave AGI of $83,208, and put the top of the 10% bracket at $24,150 instead of $24,800, arriving at $3,801.96. It then replaced that with $3,416, citing 'slightly different bracket estimates' that it never computed."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It added pre-Social Security income as $67,618 instead of $68,618, a $1,000 slip. It also used the 2025 base standard deduction of $31,500 instead of $32,200 and ended the 10% bracket at $24,000. That work produced about $3,798, but it submitted an unsupported $4,575."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It computed non-Social Security AGI as $66,131 instead of $67,131, a $1,000 subtraction error, which lowered taxable income to $34,951. It then added $2,480 and $1,218 and reported $3,080 instead of $3,698."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used the 2025 pre-OBBBA standard deduction ($30,000 plus two $1,600 aged additions, $33,200) and left out the OBBBA $6,000-per-senior deduction for both spouses. That understated deductions by $14,300 and gave taxable income of $50,251 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund and left out the $12,000 OBBBA senior deduction. Its standard deduction of $35,400 gave taxable income of $47,808 and tax of $5,241. It then submitted $3,800, citing unspecified 'additional adjustments' that it never quantified."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $31,000 base standard deduction instead of $32,200 and never applied the $6,000-per-spouse senior deduction for filers 65 and older. Taxable income came out at $49,151 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2024 tax rules, with a $33,100 standard deduction and a 10% bracket ending at $23,200, instead of 2026 law. That meant it left out the $32,200 base, the $1,650 aged additions and the $12,000 OBBBA senior deduction, and taxed $50,351 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025. It applied a pre-TCJA $16,700 standard deduction, $10,700 of personal exemptions and a 15% bracket, instead of the permanent OBBBA rules: a $32,200 standard deduction plus aged additions, the $12,000 senior deduction, and a 12% bracket."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a TCJA sunset: a $16,600 standard deduction, $10,600 of personal exemptions and a 15% rate above $23,850. It skipped the 2026 OBBBA $32,200 standard deduction and the $12,000 senior deduction, and its taxable income came out at $52,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly assumed the TCJA had expired in 2026 and used a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket. It also excluded the $243 refund. It never applied the $32,200 OBBBA standard deduction or the $12,000 senior deduction."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at about $78,818 instead of $83,451.23 and left out the $12,000 senior deduction, which gave taxable income of $46,318. Its $3,724 answer does not follow from that figure, since the 2026 brackets put tax on $46,318 at about $5,300."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no computation. Its $6,762 matches the TCJA-sunset figures other models produced with pre-TCJA personal exemptions and a 15% bracket (around $6,700 to $6,750), not the 2026 OBBBA computation: $47,500 of deductions, taxable income of $35,951.23, and tax of $3,818.15."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired and applied a $16,500 standard deduction, $10,500 of personal exemptions and 'reverted' brackets. It also dropped the $243 refund. It skipped the $32,200 standard deduction, $3,300 of aged additions and the $12,000 senior deduction that apply under 2026 law."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed that deductions, exemptions and credits eliminate all tax. In fact the $47,500 of standard and senior deductions leave $35,951.23 of taxable income on AGI of $83,451.23, and no nonrefundable credit applies to a childless couple with this AGI."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund, which gave AGI of $83,208. It applied only a $34,000 standard deduction and left out the OBBBA $6,000-per-senior deduction for both spouses, so it taxed $49,208 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund and subtracted personal exemptions, which are zero under 2026 law. It also never applied the $12,000 senior deduction. Its $5,045 answer implies taxable income of about $46,000, roughly $10,000 above the correct $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used 2025 pre-OBBBA values: a $30,000 standard deduction, $1,600 aged additions and a 10% bracket ending at $23,850. It left out the $6,000-per-senior deduction for both spouses, which put taxable income at $50,251 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $34,030 standard deduction built from a $30,750 base and $1,640 aged additions, and it left out the OBBBA $6,000 senior deduction for each spouse. Taxable income came out at $49,421 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed taxable Social Security against a nonexistent $64,000 joint threshold and got $12,085. The MFJ formula with the $32,000 and $44,000 thresholds makes the full $16,320 (85%) taxable. It also left out the two $1,650 aged additional standard deductions. The two errors partly offset each other and produced $35,016 of taxable income."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It cited 'limited itemized deductions from mortgage interest' and the SALT refund, but no mortgage interest is listed, and the $243 refund is income, not a deduction. Its $3,677 implies taxable income of about $34,775, roughly $1,176 below the correct $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as non-taxable and reported zero tax. With provisional income far above $44,000, $16,320 of the benefits is taxable, AGI is $83,451.23, and taxable income after the $47,500 of deductions is $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-TCJA' 2026 law: $29,900 of combined standard deduction, aged additions and personal exemptions, taxed at 15% above $23,900. It skipped the OBBBA-extended $32,200 standard deduction, $3,300 of aged additions, the $12,000 senior deduction and the 12% bracket."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI and its $47,500 of standard and senior deductions were correct, but it also took a $333.33 qualified-overtime deduction by treating the $1,000 by which wages exceed $25 x 40 x 52 as overtime premium. The spouse's usual weekly hours are 40, so no hours exceed the FLSA 40-hour threshold and there is no qualified overtime compensation. That cut tax by $40."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no computation. Its $5,523 implies taxable income of about $50,150, meaning roughly $33,300 of deductions: a standard deduction with aged additions but without the $12,000 OBBBA senior deduction. The correct deductions total $47,500."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it applied only a $34,030 standard deduction with aged additions and left out the $6,000-per-spouse OBBBA senior deduction. It taxed $49,421 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a 'sunset' of the TCJA: a $16,600 standard deduction, $5,300 personal exemptions and a 15% rate above $24,800. It skipped the permanent OBBBA $32,200 standard deduction and the $12,000 senior deduction, so it taxed $52,951 at 15% instead of $35,951.23 at 12%."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 state tax refund from income, which gave taxable income of $35,708 instead of $35,951.23. It also ended the 10% bracket at $24,400 instead of the 2026 MFJ figure of $24,800."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it applied only a $34,196 standard deduction and never applied the OBBBA $6,000 senior deduction for each spouse aged 65 or older. It taxed $49,255 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $1,225 of auto-loan interest, which is not allowed because no qualifying new-vehicle loan is established. It also added an invented 'temporary $1,500' to the $32,200 base standard deduction. Together these overstated deductions by $2,725 and gave taxable income of $33,226."
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score. The correct derivation gives taxable income of $35,951.23 and tax of $3,818.15."
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, standard deduction and senior deduction were correct, but it also subtracted $1,225 of auto-loan interest and a $333.33 qualified-overtime premium. With 40 usual weekly hours there is no FLSA overtime, and no qualifying vehicle loan is established. Those extra deductions cut taxable income by $1,558.33 and tax by $187."
-us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $11,400 self-employment loss and the $12,000 senior deduction and computed about $10,800 to $11,040 of tax. It then submitted -$250, a negative figure, even though tax before refundable credits has a floor of zero."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $11,400 self-employment loss and the $243 refund from AGI, which gave $94,608, and left out the $12,000 senior deduction. It computed $6,748 of tax and then submitted $5,998, which its own work does not support."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It treated taxable Social Security as a subtraction and set taxable income to zero. In fact $16,320 of Social Security is added to income, and after the $47,500 of standard and senior deductions $35,951.23 of taxable income remains, taxed at $3,818.15."
+us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Added a separate ~$925 one-child 'working family credit component' to the $1,750 child amount. Under the CWFC, a child under 18 generates only the child amount, and the working-family part is just 4% of earnings up to the base cap. That overstates the total compared with the $2,187.60 combined credit."
+us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Added a ~$950 'one-child add-on' to the working-family component. The CWFC gives extra working-family amounts only for qualifying children aged 18 to 24, not for a 6-year-old. The model's ~$3,100 therefore far exceeds the $2,187.60 credit, which is the child amount plus the capped 4%-of-earnings base."
+us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Stated that Minnesota has no refundable individual income tax credits. This misses the fully refundable Child and Working Families Credit, which pays $2,187.60 for this family with one child aged 6, $29,000 of earnings, and AGI below the phase-out threshold."
+us,scenario_032,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Found no Minnesota refundable credit and missed the Child and Working Families Credit. That credit pays the indexed per-child amount for the 6-year-old plus the capped working-family component, for $2,187.60 in total."
+us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Kept the child amount at the unindexed 2023 figure of $1,750 and guessed $410 for the working-family part. It should have used the 2026 inflation-adjusted CWFC amounts, which combine to $2,187.60 with no phase-out, so its $2,160 is too low."
+us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Reported only the unindexed $1,750 child amount and dropped the working-family part of the combined CWFC, even though $29,000 of earnings maxes out that part. It also missed the 2026 inflation adjustment, which brings the combined credit to $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Reduced the working-family part by a 'shared phaseout' that does not apply, because AGI of $30,916 is below the 2026 married-filing-jointly CWFC threshold. It also estimated rather than computing the indexed child amount, so it landed at $2,185 instead of the unreduced $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used the 2023 statutory amounts ($1,750 child + $350 working family = $2,100) without the inflation indexing Minnesota applies to the CWFC from 2024 onward, which raises the 2026 credit to $2,187.60."
+us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Calculated the pre-2023 Working Family Credit as about 35% of the federal EITC and left out the child credit. In 2023, Minnesota replaced that structure with the CWFC: an indexed per-child amount for the 6-year-old plus a working-family part of 4% of earnings, capped, for $2,187.60 in total."
+us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Assumed Minnesota's refundable credits require rent, property tax or education expenses. It missed that the CWFC is based on earnings and a qualifying child alone, giving this family with a child under 18 and $29,000 of wages $2,187.60."
+us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Stacked a ~$1,900 old-style, EITC-based Working Family Credit on top of the child credit. Under the CWFC, the working-family part for a family with no qualifying children aged 18 to 24 is only the capped 4%-of-earnings base, so the correct total is $2,187.60, not $3,700."
+us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Calculated the working-family part as 45% of an overstated $4,625 federal EITC using the repealed EITC-linked formula, then added $1,750. Under the CWFC, that part is only 4% of earnings, capped at the base, plus the indexed child amount, for $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Added a working-family amount equal to 33% of the federal EITC ($1,481) to the child credit. That pre-2023 formula no longer exists, because the CWFC's working-family part is the capped 4%-of-earnings base. This overstated the credit at $3,393 instead of $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Assigned a $1,633 Working Family Credit using the old child-based WFC schedule on top of a $1,950 child credit. Under the CWFC, the working-family part for a child under 18 is only the capped 4%-of-earnings base, so the combined credit is $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Computed a $1,368 working-family amount as 45% of the federal EITC, which is the repealed pre-2023 formula, and added the unindexed $1,750. The CWFC is the indexed child amount plus a working-family part of 4% of earnings, capped, for $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Cut the child credit to $983.60 with a phase-out even though AGI of $30,916 is below the married-filing-jointly threshold, so no reduction applies. It also set the working-family part at $550, above its cap. Its $1,533.60 understates the full $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no Minnesota refundable credit applies. This misses the CWFC, which pays $2,187.60 to a married-filing-jointly household with a child aged 6, $29,000 of earnings, and AGI below the phase-out threshold."
+us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Doubled the working-family base to $700 'for the taxpaying couple'. The CWFC's working-family part is one capped 4%-of-earnings amount per return, not per spouse. The model also left the child amount unindexed at $1,750, giving $2,450 instead of $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Added a ~$1,200 Working Family Credit sized like the old EITC-linked one-child credit. Under the CWFC, the working-family part for a family whose only child is 6 is just the capped 4%-of-earnings base, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $600 is consistent with counting only a small working-family-type amount and leaving out the CWFC child amount for the 6-year-old. That child amount alone is about $1,750 (2023 base, indexed for 2026), and the full combined credit is $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Counted only the unindexed $1,750 child amount. It left out the CWFC's working-family part, which is maxed out at $29,000 of earnings, and the 2026 inflation adjustment that together bring the credit to $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Added an implied $1,085 Working Family Credit to the $1,750 child credit. Under the CWFC, a family with no children aged 18 to 24 gets only the capped 4%-of-earnings working-family base. That overstated the total at $2,835 instead of $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $2,450 matches $1,750 plus a doubled $700 working-family base, which means it counted the base once per spouse. The CWFC allows one capped 4%-of-earnings amount per return, plus the 2026 indexed child amount, for $2,187.60."
+us,scenario_032,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Inflated the child amount at an assumed 3% a year to $1,912 but stated that no other refundable credit applies. That leaves out the working-family part of the same CWFC (4% of $29,000 of earnings, capped), which brings the combined credit to $2,187.60."
+us,scenario_032,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Applied the $29,500 phase-out threshold for non-joint filers to this married-filing-jointly household. The joint threshold is higher than $30,916 of AGI, so no phase-out applies. The model also treated the working-family part as near its phase-out end instead of at its capped base."
+us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no Minnesota refundable credit and overlooked the CWFC. That credit is the indexed child amount for the 6-year-old plus the capped working-family part on $29,000 of earnings, for $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Treated refundable credit eligibility as unspecified and returned $0, even though the household facts (a qualifying child aged 6, $29,000 of wages and AGI below the threshold) are enough to compute the $2,187.60 CWFC."
+us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used the 2023 base amounts ($1,750 child + $350 working family = $2,100) and missed the inflation indexing Minnesota applies to the CWFC after 2023, which raises the 2026 credit to $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Calculated the working-family amount as 25% of the federal EITC ($1,106.75), which is a repealed EITC-linked formula. The CWFC's working-family part is the capped 4%-of-earnings base, so the combined credit is $2,187.60, not $2,856.75."
+us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Assigned a $1,675 Working Family Credit using the old child-based WFC schedule. Under the CWFC, a child under 18 generates only the child amount, and the working-family part is the capped 4%-of-earnings base, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $3,572 is consistent with the child credit plus an old-style, EITC-linked one-child WFC of about $1,800. Under the CWFC, the working-family part is limited to the capped 4%-of-earnings base, so the combined credit is $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Left the child amount at the unindexed 2023 figure of $1,750 and used $388 for the working-family part. The 2026 inflation-adjusted CWFC amounts combine to $2,187.60, so its $2,138 is too low."
+us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Its $4,500 is consistent with a large EITC-style Working Family Credit stacked on the child credit. Under the CWFC, the working-family part for a family whose only child is 6 is just the capped 4%-of-earnings base, giving $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Added a $1,275 Working Family Credit based on the repealed child-based WFC schedule. Under the CWFC, the working-family part is the capped 4%-of-earnings base and the child amount is indexed, for $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Left the child amount at the unindexed 2023 figure of $1,750 and used $387 for the working-family part. The 2026 inflation-adjusted CWFC amounts combine to $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Counted only a ~$500 WFC-style amount and left out the CWFC child amount for the 6-year-old. That child amount is the largest part of the $2,187.60 combined credit."
+us,scenario_032,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Calculated the Working Family Credit as 40% of the federal EITC ($1,774), which is the repealed pre-2023 formula. The CWFC's working-family part is the capped 4%-of-earnings base plus the indexed child amount, for $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Estimated a $1,560 Working Family Credit using the old one-child WFC schedule on top of the $1,750 child credit. The CWFC limits the working-family part to the capped 4%-of-earnings base when the only child is under 18, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $1,577 'one-child maximum' Working Family Credit, which is the old WFC schedule. Under the CWFC, children under 18 generate only the child amount and the working-family part is the capped 4%-of-earnings base, giving $2,187.60."
+us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Computed the Working Family Credit as 33% of the federal EITC ($1,465), which is a formula Minnesota repealed when it created the CWFC. The working-family part is now the capped 4%-of-earnings base plus the indexed child amount, for $2,187.60."
+us,scenario_032,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Added an estimated $1,300 one-child Working Family Credit to the $1,750 child credit. The CWFC gives no child-based working-family increase for a child under 18, so the credit is the indexed child amount plus the capped 4%-of-earnings base, $2,187.60."
+us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return a usable value or explanation for state_refundable_credits, so no answer could be scored against the $2,187.60 CWFC."
+us,scenario_032,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Used a $1,250 'full one-child' Working Family Credit from the old WFC schedule. The CWFC's working-family part is the capped 4%-of-earnings base, and combined with the indexed child amount the credit is $2,187.60."
+us,scenario_032,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Denied the credit because the spouse had no earnings and income was supposedly above the thresholds. The CWFC requires earnings from only one spouse, and AGI of $30,916 is below the married-filing-jointly phase-out threshold. The model also never considered the child amount for the 6-year-old, so it missed the full $2,187.60."
+us,scenario_032,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Modeled the Working Family Credit with a ~$2,900 one-child maximum and a 12% phase-out, which mixes the old WFC schedule with CWFC rules. The working-family part for this family is only the capped 4%-of-earnings base, and AGI is below the phase-out threshold, so the credit is $2,187.60."
+us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Calculated only an EITC-percentage Working Family Credit (~$1,895) and left out the child amount, which is the main part of the CWFC. The correct credit is the indexed child amount plus the capped 4%-of-earnings base, $2,187.60."
+us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Used a $1,648 old-schedule Working Family Credit and added a $241 dependent care credit even though the household has no childcare expenses. It also left out the CWFC child amount, which together with the capped 4%-of-earnings base makes up the $2,187.60."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $243 state tax refund as non-taxable, which put AGI at $83,208 instead of $83,451.23 and taxable income at $35,708. It then threw out its own $3,789 result and reported an unexplained $3,699. Including the refund gives $35,951.23 of taxable income and $3,818.15 of tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI and the $47,500 standard-plus-senior deduction match, but it also took the $1,225 auto loan interest as a deduction. That deduction doesn't apply to this household. The extra $1,225 × 12% = $147 explains its $3,671.12 versus $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly treated the $11,400 self-employment loss as a capital loss capped at $3,000, when the full amount offsets ordinary income. It also used only a $30,400 basic standard deduction, leaving out the $3,300 aged additions and the $12,000 senior deduction. It then reported $5,661, which doesn't follow from its own calculation."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no workings. $2,050 implies taxable income of about $20,500, which over-deducts by roughly $15,000. The correct $83,451.23 AGI minus $47,500 of standard, aged, and senior deductions leaves $35,951.23, for $3,818.15 of tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state tax refund (AGI $83,208) and used an outdated $24,150 top for the 10% bracket instead of the 2026 MFJ $24,800. Its own result was $3,801.96, but it then swapped in an arbitrary $3,416."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It added up its pre-Social Security income as $67,618 instead of $68,618, a $1,000 error that put AGI at $82,451. It also used a $31,500 base standard deduction instead of the 2026 $32,200. Its working pointed to about $3,798, but it reported an unrelated $4,575."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It made two arithmetic errors. It computed AGI before Social Security as $66,131 instead of $67,131, understating taxable income by $1,000. It then added $2,480 + $1,218 and got $3,080 instead of $3,698. The correct $35,951.23 of taxable income gives $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It got AGI right at $83,451 but used 2025 pre-OBBBA standard deduction figures ($30,000 + 2 × $1,600) and left out the $6,000-per-senior deduction for both spouses. That put taxable income at $50,251 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state tax refund, used $1,600 aged additions, and never applied the $12,000 senior deduction. Its own working reached $5,241, and it then substituted an unexplained $3,800."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct, but it used a $31,000 base standard deduction instead of $32,200 and left out the OBBBA $6,000 deduction for each spouse aged 65+. That left taxable income at $49,151 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 2024 law to a 2026 return: 2024 standard deduction amounts, the 2024 10% bracket top of $23,200, and no $12,000 senior deduction. That produced taxable income of $50,351 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026, so it used the pre-TCJA $16,700 standard deduction, $10,700 of personal exemptions, and a 15% second bracket. OBBBA made the TCJA rates and the $32,200 standard deduction permanent and added the $12,000 senior deduction, which leaves $35,951.23 taxed at 10% and 12%."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA-sunset rules (a $16,600 standard deduction, personal exemptions, and a 15% bracket), but OBBBA kept the 10%/12% brackets and the $32,200 MFJ standard deduction for 2026. It also missed the $12,000 senior deduction."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, so it used a $16,000 standard deduction, $10,000 of personal exemptions, and a 15% bracket instead of OBBBA's $32,200 base, $3,300 aged additions, $12,000 senior deduction, and 12% bracket. It also left out the $243 state tax refund."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at $78,818 instead of $83,451.23, used a $32,500 standard deduction, and left out the aged additions and the $12,000 senior deduction. The $3,724 it reported doesn't match its own $46,318 of taxable income."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no workings. $6,762 matches the TCJA-sunset results from other models (pre-TCJA standard deduction, personal exemptions, 15% bracket) rather than OBBBA's $47,500 of deductions and 10%/12% brackets, which give $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, so it used a $16,500 standard deduction, $10,500 of personal exemptions, and pre-TCJA brackets instead of OBBBA's permanent $32,200 standard deduction, aged additions, and $12,000 senior deduction. It also left out the $243 state tax refund."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions, exemptions, and credits wipe out all tax. In fact the $47,500 standard-plus-senior deduction leaves $35,951.23 taxable out of $83,451.23 AGI, and no nonrefundable credit applies, so tax is $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state tax refund (AGI $83,208) and used a $34,000 standard deduction with no $6,000-per-senior deduction. That put taxable income at $49,208 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 refund and subtracted personal exemptions, which OBBBA repealed permanently. Its $5,045 implies about $46,000 of taxable income, so it never applied the $12,000 senior deduction on top of the $35,500 standard deduction."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct, but it used 2025 pre-OBBBA figures (a $30,000 base, $1,600 aged additions, and a $23,850 10% bracket) and left out the $12,000 senior deduction. That gave taxable income of $50,251."
+us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct, but it used a guessed $34,030 standard deduction and left out the $6,000-per-senior deduction for both spouses. That put taxable income at $49,421 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used a non-existent $64,000 threshold for Social Security taxation, which gave $12,085 taxable instead of the 85% cap of $16,320. It also left out the $3,300 in aged additions, deducting $44,200 instead of $47,500, and used $24,500 as the top of the 10% bracket."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed itemized deductions for mortgage interest, but only a mortgage balance is listed, so mortgage interest is $0 and the $47,500 standard-plus-senior deduction governs. Its $3,677 implies about $1,175 of extra deductions beyond $47,500."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as nontaxable and arrived at zero tax. Provisional income is far above the $44,000 MFJ threshold, so $16,320 is taxable, and taxable income after $47,500 of deductions is $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used post-TCJA-sunset deductions and personal exemptions totaling $29,900 and a 15% bracket. 2026 law keeps the 12% bracket and gives $47,500 of standard, aged, and senior deductions."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $1,000 gap between the spouse's $53,000 wages and $25 × 40 × 52 = $52,000 as overtime and took a $333.33 qualified-overtime deduction. With usual hours of 40 a week there are no hours above 40, so there is no qualifying overtime and the deduction is $0."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the standard deduction. $5,523 implies taxable income of about $50,000, so it left out the $12,000 senior deduction for the two spouses aged 65+."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct, but it deducted only a $34,030 standard deduction. It never applied the $6,000-per-senior deduction for both spouses, which brings total deductions to $47,500."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, so it used a $16,600 standard deduction, $5,300 personal exemptions, and a 15% second bracket instead of OBBBA's permanent $32,200 base, $12,000 senior deduction, and 12% bracket."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state tax refund, putting taxable income at $35,708 instead of $35,951.23. It also ended the 10% bracket at $24,400 instead of the 2026 MFJ $24,800."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct, but it deducted only a $34,196 standard deduction and left out the $12,000 senior deduction, which put taxable income at $49,255."
+us,scenario_033,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $1,225 of auto loan interest as an adjustment, which doesn't apply to this household. It also added a non-existent 'temporary $1,500' on top of the $32,200 base, which inflated the standard deduction to $37,000."
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer and no explanation for this variable, so there is nothing to score against $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It added two deductions that don't apply: $1,225 of auto loan interest, and $333.33 of overtime inferred from the gap between wages and the hourly rate × 40 hours. Neither applies, and they reduced taxable income to $34,392.67 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $11,400 SE loss from income and left out the $12,000 senior deduction, which led to about $10,800 of tax. It then reported an arbitrary −$250, but this tax measure can't be negative for a household with positive taxable income and no nonrefundable credits."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left the −$11,400 SSTB loss out of AGI (giving $96,095), dropped the $243 refund, and never applied the $12,000 senior deduction. It then reported $5,998, which doesn't match its own $6,748."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and a Social Security 'subtraction' reduce taxable income to zero. In fact 85% ($16,320) of benefits is taxable, and AGI of $83,451.23 minus $47,500 of deductions leaves $35,951.23 taxable."
us,scenario_033,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,043.50, then added a fabricated $64.15 “rounding adjustment.” The component calculations are already exact to the cent and require no adjustment."
-us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $6,043.50 and then improperly rounded it to $6,045. Payroll tax is requested as an annual amount, and no rule permits rounding the exact household total to a five-dollar increment."
-us,scenario_033,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model stated the correct 7.65% rate and $79,000 wage base but multiplied them incorrectly. $79,000 × 0.0765 equals $6,043.50, not $6,084."
-us,scenario_033,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the correct wage base and FICA rates but returned an arithmetic result inconsistent with them. Social Security of $4,898 plus Medicare of $1,145.50 equals $6,043.50, not $6,064."
-us,scenario_033,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model separately derived approximately $4,898 of Social Security tax and $1,146 of Medicare tax, then replaced their sum with $4,196. Using the exact Medicare amount of $1,145.50 gives total employee payroll tax of $6,043.50."
-us,scenario_033,payroll_tax,grok-4.3,llm_error,other,False,"The model applied Social Security and Medicare to the correct $79,000 wage base but rounded the resulting liability incorrectly. The exact calculation is $79,000 × 7.65% = $6,043.50, not $6,045."
-us,scenario_033,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract."
-us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $4,898 of Social Security tax and $1,145.50 of Medicare tax, then discarded that sum and submitted an unsupported $9,832 “rough estimate.” The stated components total exactly $6,043.50, with no Additional Medicare Tax or South Dakota employee payroll tax to add."
-us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly calculated each spouse's employee FICA tax and the $6,043.50 total, then improperly rounded it to $6,042. No payroll-tax computation or rounding rule produces that submitted value."
-us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct $79,000 wage base and combined 7.65% Social Security and Medicare rate but computed their product incorrectly. The product is $6,043.50, not $6,123."
+us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It correctly computed $1,612 + $377 + $3,286 + $768.50 = $6,043.50. It then added a made-up $64.15 'rounding adjustment' that has no basis in any payroll tax rule, and submitted $6,107.65."
+us,scenario_033,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It correctly computed $4,054.50 + $1,989 = $6,043.50 in FICA. It then stated 6044 and submitted 6045, rounding up for no reason instead of reporting the exact $6,043.50."
+us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"It correctly derived $1,989 + $4,054.50 = $6,043.50, then 'rounded' to 6045. That isn't valid rounding of $6,043.50, and the exact payroll tax amount was requested."
+us,scenario_033,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"Applying 7.65% to the right $79,000 wage base was correct, but the multiplication was wrong. 0.0765 × 79,000 = $6,043.50, and its $6,084 implies an effective rate of 7.7%."
+us,scenario_033,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It used the right rates (6.2% Social Security, 1.45% Medicare) on the right $79,000 base, but got the arithmetic wrong. $4,898 + $1,145.50 = $6,043.50, not $6,064."
+us,scenario_033,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It computed Social Security of $4,898 and Medicare of about $1,146, which add to about $6,044. It then threw that total away and submitted $4,196 based on an unexplained 'employee tax structure' that matches no payroll tax rule."
+us,scenario_033,payroll_tax,grok-4.3,llm_error,other,False,"It applied employee Social Security and Medicare to the correct combined wages of $79,000, but reported $6,045 instead of the exact 7.65% × $79,000 = $6,043.50, overstating the product by $1.50."
+us,scenario_033,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or reasoning for payroll_tax, so there was nothing to score. The correct answer is 7.65% employee FICA on $79,000 of wages, which is $6,043.50."
+us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"It explicitly derived $4,898 + $1,145.50 = $6,043.50 with no Additional Medicare Tax and no SD payroll tax. It then submitted 9,832 as a 'rough estimate considering all FICA', which contradicts its own calculation and suggests it mistakenly added employer-side or other FICA amounts."
+us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $1,989 + $4,054.50 = $6,043.50, then 'rounded' to $6,042. That moves the value in the wrong direction and away from the exact amount it had already found."
+us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"It named the correct rates (6.2% + 1.45%) and the correct $79,000 base, but its $6,123 equals 7.75% × $79,000. It multiplied with a mistaken combined rate instead of 7.65%, which gives $6,043.50."
us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss produces no self-employment tax, then contradicted that rule by reporting an invented negative liability. A negative SSTB amount cannot be multiplied by the 15.3% rate to create a self-employment tax credit; the correct liability is floored at zero."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the correct $16,100 standard deduction, $43,120 of taxable income and roughly $4,926 of tax from the $12,400/$50,400 brackets. It then cut the figure to $4,849 by applying 'projected bracket indexing' a second time to thresholds that were already indexed for 2026."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction instead of $16,100. It then mangled the bracket arithmetic by mixing 2024 thresholds ($11,600, $44,725) and reported $8,074, a figure that does not follow from its own 10%/12% brackets. Applying 10%/12% correctly to $43,120 gives $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model overstated the 2026 standard deduction as $16,600 instead of $16,100 and got about $4,866. It then 'reduced slightly for bracket inflation' to an unsupported $4,297, even though its $12,400/$50,400 brackets were already the indexed 2026 values."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed the exact correct chain: a $16,100 standard deduction, $43,120 of taxable income, and $1,240 + $3,686 = $4,926. It then submitted $4,969 by 'rounding to bracket estimate', an adjustment with no basis once the 2026 brackets are applied."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reached $4,926 from the $16,100 standard deduction and the 2026 10%/12% brackets. It then submitted an unexplained $5,194 as an allowance for 'bracket parameter estimates', which discarded its own correct result."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $15,750 as the 2026 single standard deduction, which is the 2025 OBBBA figure; the 2026 amount is $16,100. It also put the top of the 10% bracket at $11,950 instead of $12,400, so taxable income came to $43,470 and tax to $4,977. It also considered itemizing imputed mortgage interest, but mortgage interest is unlisted and therefore $0."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's reasoning computed exactly $1,240 + $3,686.40 = $4,926.40 from the $16,100 standard deduction and the $12,400/$50,400 brackets. It then submitted $6,499, a number that contradicts its own derivation."
-us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income of $43,820 implies a $15,400 standard deduction instead of $16,100. Its $5,020 of tax matches the 2025 10% threshold of $11,925 rather than the 2026 threshold of $12,400. Both parameters are stale, and together they overstate the tax."
-us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the 2025 pre-OBBBA $15,000 standard deduction and the 2025 10% threshold of $11,925, giving $44,220 of taxable income and $5,067.90 of tax. The 2026 values are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax."
-us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption and a 12% bracket through $50,400."
-us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction instead of $16,100 and a $12,200 top for the 10% bracket instead of $12,400. That gives $43,620 of taxable income and $4,990.40 of tax instead of $43,120 and $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above $11,600. OBBBA made the $16,100 standard deduction, the zero personal exemption and the 12% bracket through $50,400 permanent for 2026."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no working, but its $5,122 exactly matches 2024 parameters: a $14,600 standard deduction and a 10% bracket to $11,600, giving 1,160 + 12% × $33,020 = $5,122.40. It never applied the 2026 $16,100 deduction and $12,400 threshold."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired and applied an $8,350 standard deduction, a $5,050 personal exemption and a 15% bracket above about $11,450. OBBBA permanently kept the TCJA rates, set the 2026 single standard deduction at $16,100, and eliminated the personal exemption."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied pre-TCJA 2026 rules: an $8,300 standard deduction plus a $5,050 exemption, and 15% above $11,600. Under OBBBA's permanent extension, 2026 has a $16,100 standard deduction, no personal exemption and 12% up to $50,400, which gives $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model said it subtracted 'deductions and exemptions', but no personal exemption exists in 2026. Its $3,858 implies taxable income of about $34,200 under the 2026 brackets, which means it subtracted roughly $9,000 more than the $16,100 standard deduction that is the only deduction available."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15% above $11,925. OBBBA made the TCJA structure permanent, so the correct figures are a $16,100 standard deduction, no exemption, and 12% on income between $12,400 and $50,400."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied 'post-TCJA' rules: an $8,300 standard deduction, a $5,300 exemption, and 15% above $11,925. OBBBA repealed the sunset, so 2026 taxable income is $59,220 − $16,100 = $43,120, taxed at 10%/12%."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and the 2025 10% threshold of $11,925, giving $5,067.90. The 2026 values are a $16,100 deduction and a $12,400 threshold."
-us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model stated that it used the 2024 $14,600 standard deduction and the 2024 brackets (10% to $11,600) as proxies for 2026. That overstates taxable income by $1,500 and understates the 10% band compared with the 2026 $16,100 deduction and $12,400 threshold."
-us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used the 2023 $13,850 standard deduction and invented a medical-expense add-back that pushed taxable income to $50,630.50, above AGI minus the deduction. It then mixed a pre-TCJA 15% rate with a 22% bracket, and its stated terms do not even sum to $4,407. The correct figure is $43,120 taxed at 10%/12%."
-us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model gave no numeric working. Its $3,344 implies taxable income of about $29,900 under the 2026 brackets, so it subtracted roughly $13,000 more than the $16,100 standard deduction. The only deduction available is the standard deduction, which leaves $43,120."
-us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gave no computation. Its $3,720 implies taxable income of about $33,100, so it subtracted roughly $10,000 beyond the $16,100 standard deduction. The correct taxable income is $59,220 − $16,100 = $43,120, and the tax is $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model deducted 'mortgage interest' imputed from the $226,000 mortgage balance, but mortgage interest is unlisted and therefore $0. Its $4,966 implies total deductions of about $15,770, below the $16,100 standard deduction it should have taken instead."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above $12,400. OBBBA made the 12% bracket (through $50,400), the $16,100 standard deduction and the zero exemption permanent for 2026."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a 'current-law TCJA sunset' with an $8,532 standard deduction, a $5,442 personal exemption and a 15% bracket. Current law after OBBBA is a $16,100 standard deduction, no personal exemption and 12% up to $50,400."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got the $16,100 standard deduction and the $43,120 taxable income right. It then put the top of the 10% bracket at $12,200 instead of the 2026 value of $12,400, which taxed an extra $200 at 12% and overstated the tax by $4, at $4,930.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied 'pre-TCJA' 2026 rules: an $8,315 standard deduction, a $5,303 exemption, and 15% above $12,211. OBBBA made the TCJA parameters permanent, so the 2026 figures are a $16,100 deduction, no exemption and a 12% bracket through $50,400."
-us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model stated that it used the 2025 $15,000 standard deduction and the 2025 rate schedule (10% to $11,925) as an estimate for 2026. The 2026 values are $16,100 and $12,400, which give $4,926.40 rather than $5,067.90."
-us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used a $15,750 standard deduction and a $12,150 10% threshold instead of $16,100 and $12,400, and got about $4,973. It then submitted an unexplained $6,437 for 'rounding and bracket threshold adjustments', which departs from its own computation."
-us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model reached the correct $43,120 of taxable income but applied a nonexistent 22% bracket starting at $43,100; in 2026 the 12% bracket runs to $50,400. It also subtracted a Credit for the Elderly, which requires age 65 or older or disability, and then added the medical deduction back as tax. The result was an incoherent $8,440.50 instead of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the right result ($1,240 + $3,686 = $4,926 on $43,120 of taxable income) but then submitted $4,849, citing unspecified 'projected bracket indexing'. That number has no basis in its own correct 2026 parameters."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the old $15,000 standard deduction instead of the 2026 OBBBA amount of $16,100. Its arithmetic also doesn't hold together: it cites a $3,809 figure and 12% over $11,600, then reports $8,074, which is far above any 10%/12% bracket calculation on about $44,220 of taxable income. The correct tax on $43,120 is $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model used the right $16,100 deduction but never applied the brackets explicitly. It guessed '$4,950' instead of computing 10% of $12,400 plus 12% of ($43,120 − $12,400), which is $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model overstated the 2026 single standard deduction as $16,600 instead of $16,100, which gave $4,866. It then cut that by an arbitrary 'bracket inflation' adjustment to $4,297, which its own calculation doesn't support."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly computed $1,240 + $3,686 = $4,926 on $43,120 of taxable income, then 'rounded to bracket estimate' and submitted $4,969. That adjustment departs from its own correct result."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly derived $4,926 from the $16,100 deduction and the $12,400/$50,400 brackets, then submitted $5,194 as an unexplained 'rounded estimate'. Nothing in its own calculation supports that number."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $15,750 (the 2025 OBBBA amount) as the 2026 standard deduction and $11,950 as the top of the 10% bracket. The correct 2026 figures are $16,100 and $12,400. That overstated taxable income at $43,470 and gave $4,977 instead of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model computed exactly $4,926.40 ($1,240 + $3,686.40 on $43,120) and then submitted $6,499. That value contradicts its own correct derivation."
+us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model's taxable income of $43,820 implies a standard deduction of about $15,400 instead of the 2026 OBBBA amount of $16,100. Applying the 10%/12% brackets to that inflated base gave $5,020 instead of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 2025 pre-OBBBA parameters: a $15,000 standard deduction and a 10% bracket ending at $11,925. The correct 2026 figures are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax."
+us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and used an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no exemption and 10%/12% brackets."
+us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction and a 10% bracket ending at $12,200. The correct 2026 figures are $16,100 and $12,400. Its errors raised taxable income to $43,620 and gave $4,990.40 instead of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and used an $8,300 standard deduction, a $5,300 exemption and a 15% second bracket. OBBBA made the TCJA brackets and the higher standard deduction permanent, so the correct calculation is ($59,220 − $16,100) taxed at 10%/12%, which gives $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The $5,122 answer matches 2024 parameters exactly: a $14,600 standard deduction and a 10% bracket ending at $11,600, which give $1,160 + 12% × $33,020. The model needed the 2026 figures of $16,100 and $12,400, which give $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires and brought back an $8,350 standard deduction, a $5,050 personal exemption and the pre-TCJA 15% bracket. Under OBBBA, 2026 keeps a $16,100 single standard deduction with no exemption and 10%/12% brackets."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a TCJA expiry: an $8,300 standard deduction, a $5,050 exemption and a 15% rate above $11,600. OBBBA kept the TCJA structure permanently, so taxable income is $43,120 and tax is $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation and mentioned 'exemptions', which no longer exist after OBBBA. Its $3,858 understates the tax: $43,120 of taxable income ($59,220 − $16,100) taxed at 10% up to $12,400 and 12% above gives $4,926.40, and $3,858 would require a deduction of roughly $25,000."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used TCJA-expiry rules (an $8,300 standard deduction, a $5,300 exemption and a 15% bracket above $11,925), but OBBBA made the TCJA rates and the $16,100 standard deduction permanent for 2026."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed post-TCJA-expiry law, with an $8,300 standard deduction, a $5,300 personal exemption and a 15% rate. OBBBA extended the TCJA permanently, so the correct calculation uses the $16,100 deduction and 10%/12% brackets, which give $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and a 10% bracket ending at $11,925. The correct 2026 figures are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax."
+us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model explicitly used 2024 figures (a $14,600 standard deduction and a 10% bracket ending at $11,600) as stand-ins for 2026. The 2026 figures of $16,100 and $12,400 give $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model mixed a 2023 standard deduction ($13,850) with an invented medical-expense adjustment and 15%/22% brackets that don't exist under current law. Its taxable income of $50,630 is inconsistent with its own numbers. The correct calculation is $59,220 − $16,100 = $43,120, taxed at 10%/12%, which gives $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no bracket calculation. Its $3,344 understates the tax on $43,120 of taxable income, which under the 2026 brackets is $1,240 + 12% × $30,720 = $4,926.40. The figure is consistent with too large a deduction or a lower rate applied to part of the income."
+us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave no bracket calculation and asserted $3,720. Applying the 2026 $16,100 standard deduction and the 10%/12% brackets (the 10% bracket ends at $12,400) to $59,220 gives $4,926.40, so the model understated the liability."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed 'deductions including mortgage interest', but the facts list only a mortgage balance. Mortgage interest is unlisted and therefore $0, so the $16,100 standard deduction applies, giving $43,120 of taxable income and $4,926.40 of tax, not $4,966."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a TCJA expiry (an $8,300 standard deduction, a $5,300 exemption and a 15% rate above $12,400). OBBBA kept the $16,100 standard deduction and the 12% bracket up to $50,400, which gives $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used 'current-law TCJA sunset' parameters: an $8,532 standard deduction, a $5,442 exemption and a 15% rate. Under OBBBA the TCJA structure is permanent, so 2026 uses a $16,100 deduction and 10%/12% brackets."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used the correct $16,100 deduction but ended the 10% bracket at $12,200 instead of the 2026 threshold of $12,400. That taxed an extra $200 at 12% instead of 10% and gave $4,930.40 instead of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied 'pre-TCJA' 2026 rules: an $8,315 standard deduction, a $5,303 exemption and a 15% bracket. OBBBA made the TCJA brackets and the $16,100 single standard deduction permanent, which gives $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and a 10% bracket ending at $11,925 'as a practical estimate'. The 2026 figures of $16,100 and $12,400 give $43,120 of taxable income and $4,926.40 of tax."
+us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used $15,750 (the 2025 OBBBA amount) instead of the 2026 standard deduction of $16,100, and got $4,973. It then submitted an unexplained $6,437 that contradicts its own bracket calculation."
+us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed income above $43,100 at 22%, although 12% runs to $50,400 in 2026. It claimed the elderly/disabled credit, which needs the filer to be 65 or older or permanently disabled, and the head is 56. It also added a 'medical expense deduction' to the tax, although medical costs here are below the 7.5% AGI floor and a deduction can't increase tax. That produced $8,440.50 instead of $4,926.40."
us,scenario_036,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly declared the $59,220 pension income below a New Jersey adult Medicaid threshold without converting the household's MAGI to its income-to-FPL level or identifying a qualifying category. The correct computation places MAGI at 3.71 times FPL and assigns medicaid_category NONE, so the head is not eligible."
us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that payroll tax therefore equals zero, but submitted $3,715 instead. That amount applies a payroll-tax-style rate to the $59,220 taxable private pension, which is not employee compensation subject to payroll tax."
us,scenario_036,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning subtracted a $915.60 NJ medical deduction ($2,100 of premiums and expenses minus the 2% floor) that the reference does not allow, which gave $1,673.55. It then submitted value = 187, a number that matches none of its own arithmetic. The correct figure is $58,220 of taxable income (AGI less the $1,000 exemption), taxed to $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It imported a federal-style $6,500 standard deduction, which NJ does not have, and left out both the $1,000 personal exemption and the 5.525% bracket above $40,000, taxing income up to $60,000 at 3.5% for $1,417.50. It then jumped without explanation to $3,185. The correct base is $58,220 (AGI less the exemption), with 5.525% applied to $18,220 above $40,000, for $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It correctly reached $1,724.15 on $58,220 of taxable income, then threw that out to subtract an $8,405 medical deduction. That deduction counted the $7,389 employer-sponsored premium and the OTC spending in $9,589 of 'medical expenses'. The reference allows no medical deduction, so taxable income stays at $58,220 rather than $49,815."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction that included the $7,389 employer-sponsored premium, cutting taxable income to $49,815 and tax to $1,260. It then applied an invented further 'ESI premium deduction' to get $613. The reference taxes $58,220 (AGI less the $1,000 exemption) with no medical deduction, for $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated the $7,389 employer-sponsored premium, both $1,000 premium lines, and the OTC and other medical spending as $9,589 of deductible NJ medical expenses, and deducted $8,404.60 above the 2% floor. That lowered taxable income to $49,815.40 instead of the reference's $58,220, which allows no medical deduction."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It applied NJ's retirement income exclusion (up to $75,000 for single filers) to a 56-year-old who is not disabled. That exclusion requires age 62+ or disability. It then reported an unexplained 'phase-out remainder' of $1,591 instead of taxing the full $59,220 less the $1,000 exemption for $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly declined the pension exclusion, then subtracted a $1,016 medical deduction ($2,200 of premiums, OTC and other medical costs minus the 2% floor) that the reference does not allow. That put taxable income at $57,204 instead of $58,220 and the tax at $1,668 instead of $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the correct NJ single brackets to the full $59,220 but left out the $1,000 single personal exemption. That taxed $19,220 instead of $18,220 at 5.525% and gave $1,779.41 instead of $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It excluded the entire $59,220 pension under NJ's pension exclusion using only the $100,000 income test and ignored the requirement that the filer be 62+ or disabled. The head is 56, so the whole pension is taxable, which gives $58,220 of taxable income after the exemption and $1,724.16 of tax."
-us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted a $915.60 medical deduction ($2,100 of premiums and medical costs above the 2% floor of $1,184.40) that the reference does not allow. That left $57,304.40 of taxable income instead of $58,220 (AGI less the $1,000 exemption only), and $1,673.57 of tax instead of $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It correctly denied the pension exclusion, then deducted $8,404.60 of medical expenses built from $9,589 that included the $7,389 employer-sponsored premium. That dropped taxable income to $49,815.40, whereas the reference allows no medical deduction and taxes $58,220."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation beyond unnamed 'NJ-specific exemptions and deductions'. On the NJ single schedule, $1,113 corresponds to about $47,150 of taxable income, so it subtracted roughly $11,000 of deductions beyond the $1,000 exemption. Only the exemption applies, which leaves $58,220 of taxable income and $1,724.16 of tax."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It excluded the full $59,220 pension under NJ's $75,000 pension exclusion using only the income limit and never applied the age 62+ or disability requirement. The head is 56, so the pension is fully taxable and the tax on $58,220 is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It taxed only the pension 'above the NJ pension exclusion', applying a partial exclusion to a 56-year-old who is not disabled and does not qualify for it. $1,057 corresponds to about $46,150 of taxable income instead of the correct $58,220 (full pension less the $1,000 exemption), which is taxed to $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly got taxable income of $58,220 but used 2% and 2.5% for the first two NJ single brackets instead of 1.4% and 1.75%. That added $232.50 of tax and gave $1,956.66 instead of $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 NJ standard deduction (NJ has none, only the $1,000 exemption) and taxed the remainder at a flat 5.525% instead of the graduated schedule. It then subtracted a property tax credit made up as 18% of the mortgage balance, although no property tax is listed. The correct tax on $58,220 is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation. $2,860 exceeds even the NJ schedule's tax on the full $59,220 with no exemption ($1,779.41), so it used a rate schedule far above NJ's 1.4%/1.75%/3.5%/5.525% single brackets. The correct tax on $58,220 is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It subtracted unnamed 'standard credits' from NJ tax to reach about $600, but no NJ nonrefundable credit applies to this single pensioner. The tax after nonrefundable credits equals the $1,724.16 computed on $58,220 of taxable income."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $7,304.60 of medical expenses, built from $8,489 of 'qualifying' costs that included the $7,389 employer-sponsored premium, above the 2% floor. That brought taxable income down to $50,915.40, whereas the reference allows no medical deduction and taxes $58,220 for $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It excluded the entire private pension under NJ's pension exclusion on the income threshold alone, skipping the age 62+ or disability requirement that a 56-year-old head fails. The full $59,220 is taxable, which gives $1,724.16 after the $1,000 exemption."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted a medical expense deduction of about $900 (out-of-pocket premiums and expenses above the 2% floor) along with the $1,000 exemption. That lowered taxable income to about $57,320, whereas the reference allows no medical deduction, taxes $58,220, and arrives at $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It declared the head an 'eligible single filer' for NJ's pension exclusion and excluded the whole $59,220, but the exclusion requires age 62+ or disability and the head is 56. The pension is fully taxable, and the tax on $58,220 is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It cited unnamed 'deductions and credits'. $1,125 corresponds to about $47,400 of NJ taxable income, meaning roughly $10,800 of deductions beyond the $1,000 exemption that NJ does not allow here. The correct taxable income is $58,220, taxed to $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction built from $9,589 that included the $7,389 employer-sponsored premium, cutting taxable income to $49,815 instead of $58,220. It then applied an invented '$350 plus 2.75% over $20,000' schedule instead of NJ's 1.4%/1.75%/3.5%/5.525% single brackets."
-us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction from $9,589 of listed costs, including the $7,389 employer-sponsored premium, which lowered taxable income to $49,815. It then misapplied the brackets, giving $1,008.54 where the NJ schedule gives $1,259.78 on that base. The reference allows no medical deduction and taxes $58,220 for $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no substantive answer to evaluate against the $1,724.16 reference."
-us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It acknowledged that the pension exclusion is for residents 62+ and that the head is 56, but still applied an invented $100,000 income-based exclusion that wiped out the pension. Below age 62 without disability, no exclusion applies, and the tax on $58,220 is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It correctly taxed the full pension but subtracted a $1,015.60 medical deduction ($2,200 of premiums, OTC and other medical costs minus the 2% floor) that the reference does not allow. That produced $57,204.40 of taxable income and $1,668 of tax instead of $58,220 and $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning explicitly reached the correct $1,724.16 ($58,220 of taxable income after the $1,000 exemption, taxed on the NJ single brackets). It then dropped that result for an unexplained 'withholding-style flat estimation' of $2,283.06, after first also inventing a 6.37% rate above $50,000."
-us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a $16,100 'standard deduction' that NJ does not have, using it in place of the $1,000 personal exemption, and applied 5.5% instead of 5.525% on the top portion. It then added a $507.48 'property tax deduction' onto the tax even though no property tax is listed. The correct tax on $58,220 of taxable income is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted the $1,000 premium twice (as both 'health insurance premiums' and 'other premiums') to reach $2,100 of medical costs, and took a $915.60 deduction that doesn't exist because qualifying costs are below the $1,184.40 2% floor. It then submitted $187, which contradicts its own computed $1,674. The correct tax on $58,220 is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a federal-style $6,500 standard deduction that NJ doesn't have, never applied the $1,000 NJ exemption, and stretched the 3.5% bracket to $60,000 when that band covers only $35,000–$40,000, with 5.525% above it. It then abandoned its own $1,417.50 for an unsupported $3,185. Correctly, $58,220 of taxable income yields $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It correctly said the pension exclusion doesn't apply at age 56, but then guessed about $1,000 instead of running the NJ brackets. Running $59,220 − $1,000 exemption = $58,220 through the 1.4%/1.75%/3.5%/5.525% schedule gives $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It first reached the correct $1,724.15, then subtracted an $8,405 medical deduction built from $9,589 that included the $7,389 employer-sponsored premiums and counted the $1,000 premium twice. The actual qualifying out-of-pocket expenses don't exceed the 2% floor of $1,184.40, so no deduction applies and the tax stays $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It took an $8,405 medical deduction that included the $7,389 ESI premiums and counted the $1,000 premium twice, which gave $1,260. It then made a further unsupported cut to $613 based on a supposed 'ESI premium treatment'. Qualifying expenses are below the 2% floor, so taxable income is $58,220 and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added the $7,389 employer-sponsored premiums and both $1,000 premium entries into a $9,589 medical total and deducted $8,404.60 above the 2% floor. ESI premiums aren't deductible and the $1,000 premium is a single amount, so no medical deduction exists. Taxable income is $58,220, not $49,815.40, and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It applied NJ's retirement-income exclusion to a 56-year-old who isn't disabled, although the exclusion requires age 62+ or disability. It then made up a 'phase-out remainder' to land on $1,591. The full pension is taxable, and $58,220 after the exemption produces $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It built $2,200 of medical costs by counting the $1,000 premium twice and including OTC expenses, and took a $1,016 deduction. Qualifying expenses don't exceed the $1,184.40 2% floor, so taxable income is $58,220, not $57,204, and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its bracket rates were right, but it applied them to the full $59,220 and left out the $1,000 NJ personal exemption. Taxing $58,220 instead removes $55.25 and gives $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It applied the $75,000 pension exclusion based on the income limit alone and ignored that the exclusion requires age 62+ or disability. At 56, the full $59,220 is taxable, which gives $1,724.16 after the $1,000 exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted $2,100 of medical costs by double-counting the $1,000 premium and took a $915.60 deduction above the 2% floor. Qualifying expenses fall short of the $1,184.40 floor, so the only subtraction is the $1,000 exemption and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It correctly rejected the pension exclusion, but then deducted $8,404.60 of medical expenses from a $9,589 total that included the $7,389 employer-sponsored premiums and counted the $1,000 premium twice. No medical deduction clears the 2% floor, so taxable income is $58,220 and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. $1,113 implies roughly $10,000 of extra NJ deductions or exclusions on top of the $1,000 exemption, and none exist here. The correct path is $59,220 − $1,000 = $58,220, which the NJ brackets turn into $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It excluded the whole pension under NJ's $75,000 exclusion while ignoring the age-62-or-disabled requirement. The 56-year-old head doesn't qualify, so $58,220 is taxable and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It taxed only the part of the pension 'above the NJ pension exclusion', which means it gave a 56-year-old an exclusion reserved for people 62+ or disabled. With no exclusion, $58,220 is taxable and the tax is $1,724.16, not $1,057."
+us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its taxable income of $58,220 was correct, but it used 2% and 2.5% for the first two brackets, while NJ's single schedule is 1.4% on the first $20,000 and 1.75% on $20,000–$35,000. With the correct rates the tax is $1,724.16, not $1,956.66."
+us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 NJ standard deduction (NJ has none), taxed the rest at a flat 5.525% instead of the graduated brackets, and subtracted a property-tax credit computed from the mortgage balance even though no property tax is listed. The correct base is $58,220 after the $1,000 exemption, taxed at $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation, and $2,860 is about $1,136 above the graduated result. That points to the 5.525% rate (or higher) being applied to most of the income without the lower 1.4%/1.75%/3.5% bands. The correct tax on $58,220 is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It subtracted unspecified 'standard credits' that NJ doesn't offer this filer, landing at $600. Nothing reduces the bracket tax: $58,220 after the exemption gives $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $7,304.60 of medical expenses from an $8,489 base that included the $7,389 employer-sponsored premiums, which aren't deductible out-of-pocket costs. The remaining qualifying expenses are below the 2% floor, so taxable income is $58,220, not $50,915.40, and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It treated the private pension as fully excluded because income was under the threshold, skipping the requirement that the filer be 62+ or disabled. At 56, the head's $59,220 is fully taxable, which gives $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took a medical-expense deduction of roughly $930 above the 2% floor. That only works by counting the $1,000 premium twice; qualifying expenses are below the $1,184.40 floor. Without the deduction, taxable income is $58,220 and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It called the head an 'eligible' filer for NJ's pension exclusion, but the exclusion requires age 62+ or disability and the head is 56. The pension is fully taxable, and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no computation. $1,125 implies roughly $10,800 of 'deductions and credits' that don't exist here, matching the incorrect medical-deduction shortcut. The only subtraction is the $1,000 exemption, so the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted $8,405 of medical expenses using $9,589 that included the ESI premiums and the double-counted $1,000. It then used a made-up schedule ($350 + 2.75% over $20,000) instead of NJ's 1.4%/1.75%/3.5%/5.525% brackets. Correctly, $58,220 of taxable income gives $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It deducted $8,405 of medical expenses built from $9,589 that included the $7,389 ESI premiums and the $1,000 premium counted twice. It also miscomputed the bracket tax on its own $49,815 base ($1,008.54 instead of $1,259.80). With no medical deduction, $58,220 is taxable and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is nothing substantive to evaluate."
+us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It admitted the NJ pension exclusion is for residents 62+, then applied an invented $100,000 exclusion to the 56-year-old head anyway. No exclusion applies, so $58,220 is taxable and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted $2,200 of medical expenses by double-counting the $1,000 premium and including OTC costs, and deducted $1,015.60. Qualifying expenses don't exceed the $1,184.40 2% floor, so taxable income is $58,220, not $57,204.40, and the tax is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It explicitly computed the correct $1,724.16 on $58,220 of taxable income, then dropped it for an unsupported 'withholding-style flat estimation' of $2,283.06. That final figure has no basis in the NJ rate schedule."
+us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a $16,100 standard deduction, which is a federal amount and doesn't exist in NJ, instead of the $1,000 exemption. It also used a wrong 5.5% rate and then added a 'property tax deduction' to the tax. The correct base is $58,220, which gives $1,724.16."
us,scenario_036,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model said outright that the $9,629 FLSA overtime premium is not deductible, which ignores the 2026 OBBBA qualified-overtime deduction. It also subtracted the $5,789 ESI premiums from the stated wages, skipped the $18 IRA deduction, and used the 2024 $14,600 standard deduction and 2024 brackets. It then reported $1,646, a number unrelated to its own $2,308 calculation."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model got AGI ($41,539), the $16,100 standard deduction and the $12,400 10% bracket right, but it never subtracted the $9,629 qualified-overtime deduction. Taxable income should have been $15,810, not $25,439. It also reported $2,884 instead of the $2,805 its own work produced."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 OBBBA qualified-overtime deduction, so taxable income came out $9,629 too high ($25,439 instead of $15,810). It then pushed its own $2,805 result up to $3,160 on an unsupported 'recheck' of the brackets."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model identified the OBBBA overtime deduction but cut tax by only about $235. Deducting the full $9,629 at the 12% marginal rate lowers tax by $1,155, from $2,805 to $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the overtime exclusion correctly, but it used a 2025-based standard deduction ($15,000, then $15,750) and a 10% bracket of $11,925–$12,150 instead of the 2026 values of $16,100 and $12,400. It then reported $1,569, which does not match its own $1,700 result."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model never subtracted the $9,629 qualified-overtime deduction and used a 10% bracket of $12,150 instead of $12,400, which gave $2,809.68. It then reported $1,808 with no derivation. The correct taxable income is $15,810, and the tax is $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used a $15,600 standard deduction and the 2025 10% bracket of $11,925 instead of the 2026 values of $16,100 and $12,400. That inflated taxable income to $25,939."
-us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using an $8,250 standard deduction, a personal exemption and a 15% bracket. The OBBBA made the TCJA structure permanent, with a $16,100 standard deduction and a 12% bracket. The model also subtracted the $5,789 ESI premiums from wages that were already stated and left out the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used pre-TCJA 2026 parameters ($8,250 standard deduction, a $5,300 personal exemption and a 15% second bracket), but the OBBBA made the $16,100 standard deduction and 12% bracket permanent. It also left out the $9,629 qualified-overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied a TCJA-sunset regime ($8,000 standard deduction plus a $5,000 exemption, taxed at 15%) instead of the OBBBA-extended 2026 parameters ($16,100 standard deduction, 12% bracket). It also never took the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, instead of the $16,100 standard deduction and 12% bracket. It also subtracted the $5,789 ESI premiums from the stated wages and left out the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model left out both the $9,629 qualified-overtime deduction and the $18 IRA deduction. Its own stated inputs (AGI of $41,557 less a $15,800 standard deduction) produce about $2,850 of tax, not the $1,109 it reported."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset with personal exemptions, subtracted the $5,789 ESI premiums from the stated wages, and left out the $9,629 overtime deduction. It also claimed a $50 Saver's Credit, even though AGI of $41,539 is above the 2026 single eligibility limit."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used TCJA-sunset parameters ($8,500 standard deduction, a $5,150 exemption and a 15% bracket) instead of the permanent $16,100 standard deduction and 12% bracket. It also subtracted the ESI premiums from the stated wages and never took the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no working beyond 'standard deduction', and its $2,301 ignores the $9,629 qualified-overtime deduction. The figure matches subtracting ESI premiums and using older standard-deduction and bracket values. The correct taxable income is $41,539 − $16,100 − $9,629 = $15,810, and the tax is $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from wages that were already stated, left out the $9,629 qualified-overtime deduction, and used a $15,450 standard deduction and the 2025 $11,925 bracket instead of the 2026 values of $16,100 and $12,400."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the ESI premiums and never took the $9,629 qualified-overtime deduction. It also subtracted a Saver's Credit, even though AGI of $41,539 is above the 2026 single limit, so no credit applies."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used the 2025 $15,000 standard deduction and $11,925 bracket instead of the 2026 values of $16,100 and $12,400. That produced taxable income of $26,539 instead of $15,810."
-us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used the 2024 $14,600 standard deduction and $11,600 10% bracket instead of the 2026 values of $16,100 and $12,400."
-us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction and a $12,200 10% bracket instead of the 2026 values of $16,100 and $12,400. It also treated the overtime deduction as above-the-line, which lowered AGI to $31,910 and wrongly qualified the filer for a $50 Saver's Credit. The overtime deduction does not reduce AGI, so AGI stays at $41,539, above the Saver's Credit limit."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model said the standard deduction and credits wipe out the tax. In fact, $41,539 AGI less the $16,100 standard deduction and $9,629 overtime deduction leaves $15,810 of taxable income, and a childless single filer at this AGI has no nonrefundable credits, so tax is $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as a credit and assumed unspecified credits wipe out the liability. Taxable income is still $15,810 after the $16,100 standard deduction and the $9,629 overtime deduction, and no nonrefundable credit applies, so tax is $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction, did not exclude the $386 traditional 401(k) deferral, and used a $15,750 standard deduction instead of $16,100. That inflated taxable income to $26,175 instead of $15,810."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from gross wages that were already stated, which cut AGI to $35,750 instead of $41,539. It also claimed a $50 Saver's Credit, but that AGI is above the 2026 single Saver's Credit limit. Correct taxable income is $15,810, giving $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the $5,789 ESI premiums even though the stated gross wages are taxable wages, which understated AGI by $5,789. It then subtracted a $50 Saver's Credit that is unavailable at the correct $41,539 AGI."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied only the standard deduction and the 401(k)/IRA reductions and left out the $9,629 OBBBA qualified-overtime deduction. That left taxable income near $25,400 instead of $15,810."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a TCJA-sunset regime ($8,300 standard deduction, a $5,300 personal exemption and a 15% bracket) instead of the permanent 2026 values ($16,100 standard deduction and 12% bracket). It also left out the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied the overtime deduction correctly but used a $15,400 standard deduction and a $12,250 10% bracket instead of the 2026 values of $16,100 and $12,400. That overstated taxable income by $700."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model reached the correct $15,810 of taxable income, but it treated the overtime deduction as reducing AGI to $31,910 and claimed a $50 Saver's Credit. AGI is actually $41,539, which is above the 2026 single limit. It also used a $12,200 10% bracket instead of $12,400."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed 'reverted' pre-TCJA brackets with a personal exemption and a 15% rate, but the OBBBA made the $16,100 standard deduction and 12% bracket permanent. It also subtracted the ESI premiums from the stated wages and left out the $9,629 overtime deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from the stated wages, left out the $9,629 qualified-overtime deduction, and claimed a Saver's Credit that is unavailable at the correct AGI of $41,539."
-us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to grade."
-us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed $41,543 of taxable wages falls 'well below' the standard deduction and applied a personal exemption that no longer exists. The $16,100 standard deduction plus the $9,629 overtime deduction still leaves $15,810 of taxable income, so tax is $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model got AGI, the $16,100 standard deduction, the brackets and the Saver's Credit exclusion right, but it left out the $9,629 OBBBA qualified-overtime deduction. That left taxable income at $25,439 instead of $15,810."
-us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and the $18 IRA deduction, and used a $15,600 standard deduction and the 2025 $11,925 bracket instead of the 2026 values of $16,100 and $12,400."
-us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied a Child Tax Credit to a single filer with no children, which is impossible, and used it to wipe out the tax. It also left out the $9,629 overtime deduction. The correct liability, with no nonrefundable credits, is $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It stated that the $9,629 FLSA overtime premium is not deductible, so it missed the 2026 OBBBA qualified overtime deduction. It also subtracted the $5,789 ESI premiums from wages and used the 2024 $14,600 standard deduction and $11,600 10% bracket instead of $16,100 and $12,400. It computed $2,308, then submitted $1,646, a figure its own arithmetic does not support."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It counted the overtime premium twice by adding it on top of wages that already include it ($51,572). It then claimed tax is $0. In fact, after the $16,100 standard deduction and the $9,629 overtime deduction, $15,810 of taxable income remains, which produces $1,649.25 at the 10%/12% rates."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It got AGI ($41,539), the standard deduction ($16,100) and the bracket ($12,400) right, but it never subtracted the $9,629 qualified overtime deduction. That overstated taxable income at $25,439 instead of $15,810. It then raised its own $2,805 result to $2,884 with no basis."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 OBBBA qualified overtime deduction and taxed $25,439 instead of $15,810. It then raised its correctly computed $2,805 to $3,160 by an unexplained 'recheck'."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It recognized the overtime deduction but reduced tax by only about $235. Deducting the full $9,629 at the 12% marginal rate cuts tax by $1,155, from $2,805 to $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied the overtime deduction but used the 2025 $15,000/$15,750 standard deduction and an $11,925–$12,150 10% bracket instead of the 2026 values of $16,100 and $12,400. It then submitted $1,569, which contradicts its own final $1,700 computation."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It never subtracted the $9,629 qualified overtime deduction, so it taxed $25,439 instead of $15,810. It used a $12,150 10% bracket instead of $12,400. It then replaced its own $2,809.68 result with an arbitrary $1,808."
+us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 qualified overtime deduction. It also used a $15,600 standard deduction and the 2025 $11,925 bracket instead of 2026's $16,100 and $12,400, which produced $25,939 of taxable income instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA would sunset in 2026 and applied a pre-TCJA $8,250 standard deduction, a personal exemption and a 15% bracket. OBBBA made the $16,100 standard deduction and the 10%/12% brackets permanent. It also subtracted ESI premiums from wages and omitted the $9,629 overtime deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset parameters ($8,250 standard deduction, $5,300 exemption, 15% bracket) instead of the OBBBA-permanent $16,100 standard deduction and 12% bracket. It also omitted the $9,629 qualified overtime deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset ($8,000 standard deduction plus a $5,000 exemption, 15% bracket) instead of OBBBA's $16,100 standard deduction and 10%/12% brackets. It also never deducted the $9,629 qualified overtime premium."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used pre-TCJA sunset parameters ($8,300 standard deduction, $5,300 exemption, 15% rate) instead of OBBBA's $16,100 standard deduction and 12% bracket. It subtracted the $5,789 ESI premiums from wages, which lowered AGI to $35,750 instead of $41,539, and it omitted the $9,629 overtime deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 qualified overtime deduction and the $18 IRA deduction and used a $15,800 standard deduction. Its stated $25,757 of taxable income yields about $2,800 of tax, so the submitted $1,109 is inconsistent with its own inputs."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (personal exemptions, 15% bracket) instead of OBBBA's $16,100 standard deduction and 12% bracket. It subtracted ESI premiums from wages, omitted the $9,629 overtime deduction, and claimed a $50 Saver's Credit that the true $41,539 AGI rules out because it exceeds the single-filer ceiling."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters ($8,500 standard deduction, $5,150 exemption, 15% bracket) instead of OBBBA's permanent $16,100 standard deduction and 12% bracket. It also subtracted ESI premiums from wages and never deducted the $9,629 qualified overtime premium."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its reasoning mentions only wages less the standard deduction, so it omitted the $9,629 qualified overtime deduction. Subtracting that deduction lowers taxable income to $15,810 and tax to $1,649.25. Its $2,301 is consistent with taxing income near $21,000, well above that figure."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,789 ESI premiums from the stated gross wages and omitted the $9,629 qualified overtime deduction. It also used a $15,450 standard deduction and the 2025 $11,925 bracket instead of $16,100 and $12,400."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the ESI premiums and omitted the $9,629 qualified overtime deduction. It also applied a Saver's Credit, but the correct $41,539 AGI is above the single-filer Saver's Credit ceiling."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 qualified overtime deduction. It also used the 2025 $15,000 standard deduction and $11,925 10% bracket instead of 2026's $16,100 and $12,400, and so taxed $26,539 instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 qualified overtime deduction and used the 2024 $14,600 standard deduction and $11,600 bracket instead of 2026's $16,100 and $12,400. That produced $26,939 of taxable income instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It treated the overtime deduction as above the line, which lowered AGI to $31,910. That led it to claim a $50 Saver's Credit, but the deduction is taken below the line, and AGI of $41,539 exceeds the single Saver's Credit limit. It also used a $15,300 standard deduction and a $12,200 10% bracket instead of $16,100 and $12,400."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted the standard deduction and credits wipe out all liability. Even after the $16,100 standard deduction and the $9,629 overtime deduction, $15,810 of taxable income remains, and a childless single filer has no offsetting nonrefundable credits, so tax is $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated the standard deduction as a credit and assumed unspecified credits erase the liability. In fact, $15,810 of taxable income remains after the $16,100 standard deduction and the $9,629 overtime deduction, and no nonrefundable credit applies, giving $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629 qualified overtime deduction and left out the $386 traditional 401(k) exclusion. It also used the 2025 $15,750 standard deduction instead of $16,100, and so taxed $26,175 instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,789 ESI premiums from the stated gross wages, which lowered AGI to $35,750 instead of $41,539 and taxable income to $10,021 instead of $15,810. The artificially low AGI also led it to claim a $50 Saver's Credit that the true AGI rules out."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premiums from gross wages, which understated AGI at $35,750 instead of $41,539 and taxable income at $10,021 instead of $15,810. Based on that understated AGI, it applied a $50 Saver's Credit that a $41,539 AGI does not qualify for."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and the 401(k)/IRA contributions and omitted the $9,629 OBBBA qualified overtime deduction. Its $2,894 matches taxing roughly $26,000 instead of the correct $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules ($8,300 standard deduction plus a $5,300 exemption, 15% bracket) instead of OBBBA's $16,100 standard deduction and 10%/12% brackets. It also omitted the $9,629 qualified overtime deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the overtime deduction correctly but used a $15,400 standard deduction and a $12,250 10% bracket instead of 2026's $16,100 and $12,400. That overstated taxable income at $16,510 instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It reached the correct $15,810 of taxable income but used a $12,200 10% bracket instead of $12,400. It then subtracted a $50 Saver's Credit because it treated the overtime deduction as reducing AGI to $31,910. The overtime deduction does not reduce AGI, and the true $41,539 AGI exceeds the single Saver's Credit limit."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed reverted pre-TCJA brackets with a personal exemption and a 15% rate instead of OBBBA's $16,100 standard deduction and 12% bracket. It also subtracted ESI premiums from wages and omitted the $9,629 qualified overtime deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,789 ESI premiums from wages and never applied the $9,629 qualified overtime deduction. It also claimed a Saver's Credit even though the correct $41,539 AGI exceeds the single eligibility ceiling."
+us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or reasoning was returned for federal_income_tax_before_refundable_credits, so there is no answer to evaluate against the $1,649.25 result."
+us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed $41,557 of wages falls below the standard deduction and added a personal exemption that OBBBA eliminated. In fact, $15,810 of taxable income remains after the $16,100 standard deduction and the $9,629 overtime deduction, which yields $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used the correct AGI, standard deduction and brackets but never subtracted the $9,629 OBBBA qualified overtime deduction. That left taxable income at $25,439 instead of $15,810."
+us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It explicitly noted that wages include overtime but did not apply the $9,629 qualified overtime deduction. It also dropped the $18 IRA deduction and used a $15,600 standard deduction and the $11,925 bracket instead of $16,100 and $12,400."
+us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a Child Tax Credit to a single 44-year-old filer with no children, which wiped out the liability. With no qualifying child, no nonrefundable credit applies, and the $15,810 of taxable income yields $1,649.25."
us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract.
us,scenario_037,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented dependent-child credit eligibility despite the household containing only one 44-year-old adult: refundable CTC is $0 because there is no qualifying child. It also applied a child-based EITC amount even though the filer has no qualifying children, and $41,943 exceeds the childless EITC income limit, so EITC is $0."
us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated North Carolina’s adoption of Medicaid expansion as sufficient for eligibility and failed to compare the head’s MAGI income of 2.60 times FPL with the expansion-adult income limit. The head exceeds that limit and has no alternative Medicaid pathway, yielding medicaid_category NONE and ineligibility."
-us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It used the correct $41,943 base and correct rates, but it got Medicare wrong: 1.45% × $41,943 is $608.17, not $608.27. It then added an unexplained 'minor adjustment' that raised its own $3,208.74 to $3,211.64, which no rule supports. The correct total is $2,600.49 + $608.18 = $3,208.67."
-us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the $5,789 employer-sponsored insurance premium as a pre-tax deduction and cut FICA wages to $36,154. The facts list no cafeteria-plan salary reduction, so Social Security and Medicare apply to the full $41,943 of gross wages. Applying 7.65% to that base gives $3,208.67, not $2,765.78."
-us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted the $5,789 employer-sponsored insurance premium from wages as a pre-tax health premium, leaving $36,154. The payroll-tax wage base is the full $41,943 of gross wages. Leaving out $5,789 of taxable wages understated the tax by about $442.89."
-us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It excluded the $5,789 employer-sponsored insurance premium from FICA wages and applied 6.2% and 1.45% to $36,154. No employee salary-reduction election is listed, so both taxes apply to the full $41,943: Social Security $2,600.49 and Medicare $608.18."
-us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages by the $5,789 employer-sponsored insurance premium, as if it were a Section 125 pre-tax deduction, and taxed only $36,154. Payroll tax applies to the full $41,943 of gross wages, which gives $3,208.67."
-us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It computed FICA on gross wages minus 'pre-tax ESI premiums' ($36,154). The $5,789 premium is not a salary reduction in the stated facts, so the 7.65% combined rate applies to the full $41,943, giving $3,208.67."
-us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It correctly found about $2,601 in Social Security and $608 in Medicare on the full $41,943, totaling about $3,209. It then lowered the total to $3,151 without any rule to justify it. No deduction or state payroll adjustment applies, so the answer is $3,208.67."
-us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It correctly kept 401(k) contributions and the overtime deduction in FICA wages, but it wrongly subtracted the $5,789 employer-sponsored insurance premium to get $36,154. Nothing in the facts makes that premium a pre-tax payroll deduction, so the base is the full $41,943 and the tax is $3,208.67."
-us,scenario_037,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It applied 7.65% to $36,154 after taking out the $5,789 employer-sponsored insurance premium, treating it as a cafeteria-plan exclusion that the facts do not include. Employee Social Security and Medicare apply to the full $41,943 of gross wages, which yields $3,208.67."
-us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"It used the full-wage base and the standard 6.2% + 1.45% employee rates, but it rounded the result to $3,210 instead of reporting the exact $3,208.67. Its own base calls for $2,600.49 in Social Security plus $608.18 in Medicare."
-us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $5,789 employer-sponsored insurance premium as pre-tax ESI, leaving $36,154 of FICA wages, and then rounded each component to whole dollars to get $2,766. The FICA base is the full $41,943 of gross wages, which gives $3,208.67."
-us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It excluded the $5,789 employer-sponsored health premium from FICA wages as a pre-tax deduction, which the facts do not support, and taxed $36,154. Social Security at 6.2% and Medicare at 1.45% apply to the full $41,943, for a total of $3,208.67."
-us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is 7.65% of the full $41,943 of wages, which is $3,208.67."
-us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"It used the correct $41,943 base and correct rates, getting $2,601 in Social Security and $608 in Medicare. It then added these to $3,213 instead of $3,209. The exact total is $2,600.49 + $608.18 = $3,208.67."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model used the outdated $10,750 deduction and a 4.5% rate to reach $1,386. It then zeroed the liability by pointing to nonrefundable credits that do not exist, because no NC nonrefundable credit applies to this childless single filer. The correct result is $28,789.45 of taxable income × 3.99% = $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model first had nearly the right figure (about $1,149). It then reversed itself by assuming the stated gross wages already excluded the 401(k) deferral, so it taxed $29,193. The stated wages are gross, so the $386 401(k) and $18 IRA contributions must be subtracted to reach AGI of $41,539.45."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model had the correct $28,789 taxable income but applied a 4.0% rate instead of the 2026 NC rate of 3.99%. It then added an unexplained 'traditional contribution effects' adjustment to reach $1,180, when no such NC adjustment exists."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model computed the correct $28,789 × 3.99% ≈ $1,149. It then cut this to $883 by citing an NC 'decoupling/deduction treatment' that does not apply to this wage earner, since no further NC subtraction reduces the $28,789.45 of taxable income."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model reached the correct $28,789 × 3.99% = $1,148.68 at one point but kept switching standard deduction amounts. It finally settled on an invented $21,500 'doubled' deduction and a 401(k)-only AGI. The 2026 NC single standard deduction is $12,750."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of the 2026 rate of 3.99% on the correct $28,789 base, which gave $1,223.5. It then raised this to $1,685 for unexplained 'rounding and updates'. The correct figure is $28,789.45 × 3.99% = $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $13,750 NC standard deduction instead of the 2026 single amount of $12,750. That understated taxable income by $1,000 ($27,789 instead of $28,789.45)."
-us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI subtracts the $5,789 employer-sponsored insurance premiums from the stated gross wages. Those premiums do not reduce AGI here, so AGI is $41,539.45 (wages less only the 401(k) and IRA contributions)."
-us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied an invented $15,400 NC standard deduction instead of the 2026 single amount of $12,750. That understated taxable income, which is $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 employer-sponsored insurance premiums from wages to get an AGI of $35,750, when the correct AGI is $41,539.45. It also used a $12,850 standard deduction instead of NC's $12,750."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $1,787.52 answer is about 4.3% of nearly the whole $41.5k AGI. The model left out the $12,750 NC standard deduction and used a rate higher than the 2026 flat rate of 3.99%."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from a federal AGI of $35,750, which subtracts the $5,789 employer-sponsored insurance premiums from gross wages. Those premiums do not reduce AGI, so NC taxable income is $41,539.45 − $12,750 = $28,789.45, not $23,000."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's $35,750 state AGI removes the $5,789 employer-sponsored insurance premiums from wages. Only the $386 401(k) and $18 IRA contributions reduce AGI, which makes taxable income $28,789.45 rather than $23,000."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model applied an outdated rate above the 2026 NC flat rate of 3.99%; it stated 4.25%. Its $1,281 answer equals the correct $28,789 base taxed at about 4.45%, when the correct result is $28,789.45 × 3.99% = $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $35,750 starting AGI subtracts the $5,789 employer-sponsored insurance premiums from gross wages. Those premiums do not reduce AGI here, so the NC base is $41,539.45 − $12,750 = $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $917.70 answer implies $23,000 of taxable income, which comes from a $35,750 AGI that subtracts the $5,789 employer-sponsored insurance premiums. The correct AGI is $41,539.45, which gives $28,789.45 of taxable income."
-us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model had the correct AGI of $41,539 and the correct 3.99% rate, but used an invented $15,500 standard deduction instead of NC's 2026 single amount of $12,750."
-us,scenario_037,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model subtracted the $9,629 overtime premium to get a $31,910 AGI. The federal overtime deduction is a below-the-line deduction that does not reduce the federal AGI NC starts from, so AGI stays at $41,539.45."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The correct derivation is $28,789.45 × 3.99% = $1,148.70. The $1,268 answer is about $119 higher, which matches applying an outdated rate of about 4.4% to the correct base instead of the 2026 rate of 3.99%."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model claimed NC deductions and credits wipe out the liability. But $41,539.45 of AGI far exceeds the $12,750 standard deduction, and NC has no nonrefundable credit for this childless filer, so tax is $28,789.45 × 3.99% = $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly calculated $1,148.68 and then misrounded it to $1,147. Rounding $1,148.68 to the nearest dollar gives $1,149, and the unrounded figure is $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $19,160 taxable income comes from a $31,910 AGI that subtracts the $9,629 overtime premium. The federal overtime deduction is below the line and does not reduce the AGI NC starts from, so taxable income is $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model correctly left out the overtime deduction but started from a $35,750 AGI that subtracts the $5,789 employer-sponsored insurance premiums. The correct AGI is $41,539.45, which gives $28,789.45 of taxable income."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI removes the $5,789 employer-sponsored insurance premiums from gross wages. Only the $386 401(k) and $18 IRA contributions reduce AGI, so the correct base is $41,539.45 − $12,750 = $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The $1,370 answer equals $30,789 × 4.45%, which means the model used the outdated $10,750 standard deduction and a 4.45% rate. The 2026 values are $12,750 and 3.99%."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model started from a $31,910 AGI that subtracts the $9,629 overtime premium. The federal overtime deduction is below the line and does not reduce the federal AGI NC starts from, which is $41,539.45."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model treated federal AGI as already reflecting the overtime deduction ($31,910). The overtime deduction is taken below the line, so AGI and NC's starting point stay at $41,539.45."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI subtracts the $5,789 employer-sponsored insurance premiums, which do not reduce AGI; the correct AGI is $41,539.45. It also used 4% instead of the 2026 NC rate of 3.99%."
-us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's $35,750 federal AGI removes the $5,789 employer-sponsored insurance premiums from gross wages. AGI is actually $41,539.45, which gives $28,789.45 of taxable income instead of $23,000."
-us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the $5,789 employer-sponsored insurance premiums as a pre-tax reduction to wages. Those premiums do not reduce AGI here, so only the 401(k) and IRA contributions come out, and AGI is $41,539.45."
-us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed taxable income was near zero after the standard deduction, but $41,539.45 of AGI less the $12,750 deduction leaves $28,789.45 taxable. It also used a 4.5% rate instead of the 2026 rate of 3.99%."
-us,scenario_037,state_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model's explanation correctly derives $28,789 × 3.99% = $1,148.68. The value it submitted, $1,223.53, equals $28,789 × 4.25%, so the number does not match its own derivation and uses an outdated rate."
-us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of the 2026 rate of 3.99%, and it left out the $18 IRA deduction. It then replaced its own $1,296 result with an unexplained $1,773.56."
-us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The $1,234 answer is $85 above the correct $1,148.70, which is the size of error produced by using the outdated $10,750 standard deduction ($30,789 × 3.99% ≈ $1,228) or a 4.25% rate. The correct values are the $12,750 deduction and 3.99% applied to $41,539.45 of AGI."
-us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model applied only a 200% FPL CHIP upper income limit and concluded that $22,993 for a household of four being under it, plus age 7 being under 19, established CHIP eligibility. It never ran the Medicaid screen first: at roughly 71% FPL the child falls under Louisiana's Medicaid threshold for ages 6–18 and qualifies under the OLDER_CHILD category, and CHIP covers only children who do not qualify for Medicaid."
-us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model tested child1 against a single ~250% FPL LaCHIP ceiling and treated ""well below threshold"" as qualifying, which inverts the CHIP test. Income far below that ceiling places the child under Louisiana's children's Medicaid limit for ages 6–18, making them Medicaid-eligible under the OLDER_CHILD category, and Medicaid eligibility bars CHIP."
-us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model refined the income figure (wages plus self-employment net of the SE tax adjustment) but used it only against the ~250% FPL upper CHIP limit, concluding that a dependent child inside that limit is CHIP-eligible. The missing step is the Medicaid floor: at roughly 71% FPL the 7-year-old qualifies for Louisiana Medicaid under the OLDER_CHILD category, which makes CHIP eligibility 0 because CHIP covers only children Medicaid does not."
-us,scenario_038,child1_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model built a complete upper-bound test — age under 19, no other coverage, income $22,993 against 250% FPL ≈ $80,375 — and never checked whether income was low enough for Medicaid instead. Louisiana covers children ages 6–18 under Medicaid up to 142% FPL, and this household sits near 71% FPL, so child1 is Medicaid-eligible under the OLDER_CHILD category and is therefore outside CHIP's residual population."
-us,scenario_038,child1_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model collapsed the two programs into one ""LaCHIP/Medicaid"" bucket and submitted 1 for CHIP on the strength of the child qualifying for children's coverage at 212–250% FPL levels. Its own statement that child1 ""qualifies for LaCHIP/Medicaid"" identifies the disqualifier: Medicaid eligibility under the OLDER_CHILD category at ~71% FPL excludes the child from CHIP, which is defined as coverage for children who do not qualify for Medicaid."
-us,scenario_038,child1_chip_eligible,gemini-3.5-flash-lite,llm_error,health_coverage,False,"The model gave only an income-and-family-size screen against a combined ""Louisiana Medicaid/CHIP"" limit and returned 1, a result consistent with testing income against the CHIP ceiling alone. The correct derivation runs Medicaid first: ~$22,992 for four people is about 71% FPL, under Louisiana's Medicaid limit for ages 6–18, so child1 is Medicaid-eligible as an OLDER_CHILD and CHIP eligibility is 0."
-us,scenario_038,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for child1_chip_eligible, so the key was absent from its outputs rather than substantively wrong. The correct derivation yields 0: at ~71% FPL the 7-year-old qualifies for Louisiana Medicaid under the OLDER_CHILD category, and CHIP covers only children who do not qualify for Medicaid."
-us,scenario_038,child1_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model computed income correctly at about 73% FPL for a family of four and explicitly invoked a combined ""CHIP/Medicaid income threshold"" well above 200% FPL, then treated falling below it as CHIP qualification. That income level is exactly what makes the child Medicaid-eligible under Louisiana's OLDER_CHILD category for ages 6–18, and Medicaid eligibility forecloses CHIP, which is limited to children who do not qualify for Medicaid."
+us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly computed 6.2% and 1.45% of $41,943 and got a total of $3,208.74. It then swapped in $3,211.64 for unexplained 'minor adjustments,' adding about $3 that no payroll tax rule supports."
+us,scenario_037,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"The model named the right rates (6.2% Social Security and 1.45% Medicare on gross wages) but submitted 0 as a placeholder and never applied them. Applying 7.65% to the $41,943 of wages gives about $3,208.67."
+us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $5,789 of employer-sponsored insurance premiums from wages and used a FICA base of $36,154. Nothing in the facts says the premiums are paid through a pre-tax cafeteria plan, so FICA applies to the full $41,943, which gives $3,208.67."
+us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $5,789 of ESI premiums as a pre-tax payroll deduction and taxed only $36,154. That assumes a Section 125 salary-reduction election the household facts don't list. Social Security and Medicare apply to the full $41,943 of gross wages."
+us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model left the $5,789 of employer-sponsored insurance premiums out of FICA wages and applied 7.65% to $36,154. The payroll tax base is the full $41,943 of gross wages, so Social Security is $2,600.49 and Medicare is $608.18."
+us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model cut the FICA base to $36,154 by treating ESI premiums as pre-tax. The facts include no cafeteria-plan salary reduction, so 6.2% and 1.45% apply to the full $41,943, for a total of $3,208.67."
+us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model netted the $5,789 of ESI premiums out of wages before applying 7.65%, understating the base by $5,789. Employee FICA applies to all $41,943 of gross wages."
+us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly reached about $2,601 + $608 = $3,209 on the full $41,943 of wages. It then reported $3,151 with no rule behind the $58 cut, which contradicts its own arithmetic."
+us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) contributions in FICA wages but wrongly took out the $5,789 of employer-sponsored insurance premiums, taxing only $36,154. No pre-tax premium arrangement is listed, so the base is the full $41,943, which gives $3,208.67."
+us,scenario_037,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $36,154 after taking out the $5,789 of ESI premiums, assuming a Section 125 salary reduction the facts don't include. The FICA base is the full $41,943 of gross wages."
+us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"The model used the right method, the employee share of Social Security and Medicare on gross wages, but reported a rounded $3,210 instead of the exact amount. Exactly, 6.2% gives $2,600.49 and 1.45% gives $608.18, for $3,208.67, so the answer overshoots by $1.33."
+us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $5,789 of ESI premiums as pre-tax, taxed $36,154, and rounded each piece to whole dollars. Payroll tax applies to the full $41,943 of gross wages, which gives $3,208.67."
+us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model treated the $5,789 of employer-sponsored health premiums as excluded from FICA wages and computed tax on $36,154. Without a listed cafeteria-plan salary reduction, Social Security and Medicare apply to the full $41,943."
+us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value and no reasoning, so its answer couldn't be parsed. The correct value is 7.65% of $41,943 in gross wages, or $3,208.67."
+us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed about $2,601 in Social Security and $608 in Medicare on the full $41,943, but added them to $3,213 instead of $3,209. That is an arithmetic slip on top of a correct method."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used the outdated $10,750 NC standard deduction instead of $12,750 and a 4.5% rate instead of 2026's 3.99%. It then set its own $1,386 figure to $0 by citing nonrefundable credits that do not exist for this filer. The correct calculation is 3.99% × $28,789.45 = $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"Submitted a $0 placeholder and never calculated NC tax. The correct steps are AGI of $41,539.45 minus the $12,750 standard deduction, taxed at the 3.99% flat rate, which gives $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Got about $1,149 at first, then reversed itself by assuming the listed gross wages already excluded the traditional 401(k) contribution. It dropped the $386 401(k) and $18 IRA reductions and taxed $29,193 instead of $28,789.45. The listed amount is gross wages, so both contributions reduce AGI."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Found the correct $28,789 NC taxable income but used a 4.0% rate instead of the 2026 rate of 3.99%. It then added an unexplained bump to $1,180 for 'traditional contribution effects' that NC does not apply."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Correctly reached $28,789 × 3.99% ≈ $1,149, then cut it to $883 by citing NC 'decoupling/deduction treatment.' No NC addition or subtraction applies to this wage-only filer, so the made-up adjustment caused the error."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Computed the correct $1,148.68 partway through, then based its final answer on an invented $21,500 2026 NC standard deduction and left out the $18 IRA deduction. That gives $20,057 × 3.99%. The actual NC single standard deduction is $12,750."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Used a 4.25% rate (the 2025 rate) instead of 2026's 3.99% on the correct $28,789 base. It then raised the result to $1,685 with no stated calculation."
+us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used a $13,750 NC single standard deduction instead of $12,750. That understated taxable income by $1,000, giving $27,789 × 3.99% = $1,109 instead of $28,789.45 × 3.99% = $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from an AGI of $35,750, which means it subtracted the $5,789 employer-sponsored insurance premium from gross wages. The listed gross wages are reduced only by the traditional 401(k) and IRA amounts, so AGI is $41,539.45 and NC taxable income is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used a $15,400 NC single standard deduction instead of the actual $12,750, so taxable income came out at $26,139 instead of $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,789 employer-sponsored insurance premium from wages to get an AGI of $35,750, and also used a $12,850 standard deduction instead of $12,750. The correct NC taxable income is $41,539.45 − $12,750 = $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Applied a made-up 4.3% rate to about $41,570, which is roughly full AGI with no $12,750 NC standard deduction taken off. The correct calculation is 3.99% on $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,750 by treating the $5,789 employer-sponsored insurance premium as a pre-tax wage exclusion. Only the $386 traditional 401(k) and $18 IRA reduce the $41,943 gross wages, so AGI is $41,539.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Started from a state AGI of $35,750 that removes the $5,789 employer-sponsored insurance premium from wages. AGI is $41,539.45, so taxable income is $28,789.45, not $23,000."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Used a 4.25% NC rate instead of the 2026 flat rate of 3.99%, and its $1,281 implies a taxable base of about $30,141 instead of $28,789.45. The answer is wrong on both the rate and the base."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Started from an NC AGI of $35,750, which subtracts the $5,789 employer-sponsored insurance premium. Gross wages are reduced only by the $404 in traditional 401(k)/IRA contributions, so AGI is $41,539.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $917.70 equals 3.99% × $23,000, meaning NC AGI was $35,750 after subtracting the $5,789 employer-sponsored insurance premium. The correct AGI is $41,539.45, which gives a $28,789.45 base."
+us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Had the correct $41,539 AGI and 3.99% rate but used a $15,500 NC single standard deduction instead of $12,750."
+us,scenario_037,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Reduced federal AGI by the $9,629 overtime premium to get $31,910. The federal qualified-overtime deduction comes off federal taxable income, not AGI, so it never enters NC's AGI-based starting point. NC taxable income is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Gave no derivation. The correct calculation is 3.99% × ($41,539.45 − $12,750) = $1,148.70. Its $1,268 matches applying about 4.4% to the correct base, or taking too small a standard deduction."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Claimed wage income was low compared with NC standard deductions and credits. In fact the $41,539 AGI is far above the $12,750 single standard deduction, and no NC nonrefundable credit applies, leaving $28,789.45 taxable at 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"Its reasoning reached the correct $28,789 × 3.99% = $1,148.68. It then submitted $1,147, a rounding/transcription slip that does not match its own result."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $19,160 taxable income comes from subtracting the $9,629 federal overtime-premium deduction before the NC standard deduction. That deduction is below the line, outside federal AGI, so it does not reduce NC's base of $41,539.45 − $12,750 = $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Correctly excluded the overtime deduction from the NC base but started from an AGI of $35,750 by subtracting the $5,789 employer-sponsored insurance premium from gross wages. AGI is $41,539.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,750, which treats the $5,789 employer-sponsored insurance premium as a wage exclusion. Only the traditional 401(k) and IRA contributions reduce gross wages, so AGI is $41,539.45 and taxable income is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Gave no derivation. Its $1,370 matches $30,789 × 4.45%, meaning it used the outdated $10,750 standard deduction and an above-2026 rate, instead of $12,750 and 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Started from an AGI of $31,910 that subtracts the $9,629 overtime premium. The federal overtime deduction does not reduce AGI, so the NC base is $41,539.45 − $12,750 = $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Explicitly treated the overtime deduction as already reflected in federal AGI ($31,910). The qualified-overtime deduction is taken below the line, after AGI, so it never reaches NC's AGI starting point."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Started from an AGI of $35,750 that subtracts the $5,789 employer-sponsored insurance premium, and used a 4% rate instead of 3.99%. The correct base is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used a federal AGI of $35,750, which removes the $5,789 employer-sponsored insurance premium from gross wages. AGI is $41,539.45, so NC taxable income is $28,789.45, not $23,000."
+us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Explicitly subtracted the $5,789 employer-sponsored insurance premium as a pre-tax exclusion from the listed gross wages. Only the $386 traditional 401(k) and $18 IRA reduce AGI, which is $41,539.45."
+us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Used a 4.5% rate instead of 3.99% and claimed taxable income was near zero. AGI of $41,539 minus the $12,750 standard deduction leaves $28,789.45 taxable."
+us,scenario_037,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its explanation computes the correct $1,148.68, but the submitted value of $1,223.53 equals $28,789 × 4.25%. The numeric answer therefore used the 2025 rate instead of 2026's 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Used a 4.5% rate instead of 3.99% and left out the $18 IRA deduction, getting $1,296.32. It then submitted an unrelated $1,773.56 that does not follow from any step it showed."
+us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Gave no derivation. The correct calculation is 3.99% × ($41,539.45 − $12,750) = $1,148.70. Its $1,234 matches applying about 4.29% to the correct base, meaning it used a rate above 2026's 3.99%."
+us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model found income of $22,993 below 200% FPL and called Child 1 CHIP-eligible. It never applied the rule that a child who qualifies for Medicaid is excluded from CHIP. At about 73% FPL, the 7-year-old falls in Louisiana's OLDER_CHILD Medicaid category, so CHIP eligibility is 0."
+us,scenario_038,child1_chip_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model concluded CHIP eligibility from 'under 19 in a low-income household' alone. It skipped the step that sends children with income below the Medicaid limit to Medicaid instead of CHIP. Child 1's income of about 73% FPL qualifies them for Medicaid as an older child, which rules out CHIP."
+us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model compared income only against a roughly 250% FPL CHIP ceiling. It did not recognize that CHIP covers only children who fail the Medicaid income test. At about 73% FPL, Child 1 meets the Louisiana OLDER_CHILD Medicaid limit, so they are Medicaid-eligible and not CHIP-eligible."
+us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being under the CHIP limit of about 250% FPL as enough for CHIP eligibility. It ignored the lower Medicaid tier that the child's income clearly meets. Because Child 1 is eligible for Medicaid as an OLDER_CHILD, PolicyEngine excludes them from CHIP."
+us,scenario_038,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age and income against a 250% FPL LaCHIP ceiling and treated 'no other coverage' as the only bar to CHIP. It never checked Medicaid eligibility first. At about 73% FPL, the 7-year-old qualifies for Medicaid in the OLDER_CHILD category, so CHIP eligibility is 0."
+us,scenario_038,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model merged 'LaCHIP/Medicaid' into one program and so counted the child's Medicaid-level income as CHIP eligibility. PolicyEngine treats them as separate tiers: a child under the Medicaid limit, here an OLDER_CHILD at about 73% FPL, is Medicaid-eligible and therefore not CHIP-eligible."
+us,scenario_038,child1_chip_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted CHIP eligibility from income and family size without noting that the child's income, about 73% FPL, meets the Medicaid limit for children aged 6–18. Medicaid eligibility in the OLDER_CHILD category excludes Child 1 from CHIP."
+us,scenario_038,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child1_chip_eligible, so nothing could be scored. The correct answer is 0, because Child 1 is Medicaid-eligible in the OLDER_CHILD category, which excludes them from CHIP."
+us,scenario_038,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly put household income at about 73% FPL but then used a single combined 'CHIP/Medicaid' threshold and called the child CHIP-eligible. At that income the 7-year-old meets the Medicaid OLDER_CHILD test, and Medicaid eligibility excludes them from CHIP."
us,scenario_038,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,The model recognized that age seven exceeds Head Start’s typical preschool range of ages three through five but then incorrectly treated income eligibility as overriding the age requirement. A seven-year-old is not eligible for this preschool-age Head Start output.
us,scenario_038,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly stated that Head Start serves preschool-age children ages three through five, yet returned eligible for a seven-year-old based solely on household income. The unmet age condition independently requires a value of 0."
us,scenario_038,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model explicitly and incorrectly classified age seven as within the Head Start preschool-age range. Head Start’s preschool pathway covers younger children, so Child 1 fails the age requirement regardless of the household’s income."
us,scenario_038,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's Medicaid older-child pathway to the 7-year-old. At household MAGI of 0.66 times FPL, the child satisfies the category's 2026 income limit, and the bank account and vehicle values are irrelevant to this MAGI test."
us,scenario_038,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age seven as within WIC’s eligible child age range. WIC covers children under age five, so Child 1 fails the categorical age test."
-us,scenario_038,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model built MAGI of $22,993 and tested it only against an approximate 202% FPL CHIP ceiling (and a nonexistent CHIP asset test), never applying the Medicaid children's MAGI screen that must be run first. At roughly 71% of FPL for a household of four, child2 falls inside the OLDER_CHILD (ages 6–18) Medicaid category, and CHIP by design covers only children who do not qualify for Medicaid, so the very margin the model cited as proof of eligibility is what disqualifies the child."
-us,scenario_038,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied one test — ~$22,993 against a ~250% FPG Louisiana CHIP limit — and stopped, omitting the Medicaid-first screen that governs CHIP. Income at about 71% of FPL for a family of four puts the 6-year-old squarely in the OLDER_CHILD Medicaid category, and Medicaid eligibility forecloses CHIP."
-us,scenario_038,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model reasoned solely that ~$23K for a family of four is ""well within CHIP income limits"" and never checked whether the child first qualifies for Medicaid. That income is roughly 71% of FPL, inside the OLDER_CHILD (ages 6–18) Medicaid MAGI limit, and CHIP is residual to Medicaid, so the child is CHIP-ineligible."
-us,scenario_038,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model used an uninsured-plus-250%-FPL screen and treated the absence of listed coverage as satisfying CHIP's non-Medicaid condition, when the actual condition is not qualifying for Medicaid. With MAGI of $22,993 against a family-of-four FPL of about $32,150 (roughly 71% FPL), child2 qualifies under the OLDER_CHILD Medicaid category, which bars CHIP regardless of the CHIP income ceiling."
-us,scenario_038,child2_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model netted self-employment income to roughly $21,800, counted the 18-year-old spouse as a fifth household member, and compared the result to a 212% FPL LaCHIP ceiling — a CHIP-only income test with no Medicaid screen ahead of it. Countable income far below the CHIP ceiling puts the 6-year-old inside the OLDER_CHILD Medicaid category, and CHIP covers only children who fail to qualify for Medicaid, so the answer is no."
-us,scenario_038,child2_chip_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave only ""eligible based on household income and family size,"" an answer consistent with comparing household income to a Louisiana CHIP income ceiling and nothing else. The correct derivation screens Medicaid first: at about 71% of FPL for a family of four, child2 qualifies under the OLDER_CHILD (ages 6–18) Medicaid category, and Medicaid eligibility excludes the child from CHIP."
-us,scenario_038,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model verified only two conditions — age under 19 and $22,993 below a ~250% FPL CHIP threshold — and skipped the Medicaid-first screen that determines CHIP eligibility. That income is roughly 71% of FPL for a household of four, well inside the OLDER_CHILD Medicaid MAGI limit, so child2 is Medicaid-eligible and therefore not CHIP-eligible."
+us,scenario_038,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared the $22,993 MAGI only against a CHIP ceiling of about 202% FPL. It never applied the rule that a child who is income-eligible for Medicaid is excluded from CHIP. At this income, Child 2 qualifies for Medicaid under the older-child (age 6–18) category, so CHIP eligibility is 0."
+us,scenario_038,child2_chip_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated 'child under 19 in a low-income household' as enough for CHIP. It ignored that CHIP covers only children who are not Medicaid-eligible. With income this low, Child 2 is Medicaid-eligible under the older-child category, which makes CHIP eligibility 0."
+us,scenario_038,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model checked about $23K of income only against an upper CHIP limit of about 250% FPG. It never tested the lower Medicaid limit for children aged 6–18, which this household meets. Child 2 is therefore covered by Medicaid and excluded from CHIP."
+us,scenario_038,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that income 'well within CHIP income limits' means CHIP eligibility. It missed that CHIP is the program for children above the Medicaid limit. Income of about $23K for a family of four places Child 2 in Medicaid's older-child category, so the CHIP value is 0."
+us,scenario_038,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model listed the CHIP criteria as age under 19, uninsured, and income under 250% FPL. It omitted the condition that the child must not be Medicaid-eligible. At about 72% FPL, Child 2 is Medicaid-eligible as an older child (ages 6–18), which disqualifies the child from CHIP."
+us,scenario_038,child2_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model compared about $21,800 of countable income with LaCHIP's 212% FPL ceiling and stopped there. It never recognized that income this far below the children's Medicaid limit puts Child 2 in Medicaid's older-child category. Under PolicyEngine's rules, that category excludes the child from CHIP."
+us,scenario_038,child2_chip_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted CHIP eligibility from income and family size without distinguishing Medicaid from CHIP. At this household's very low income, Child 2 qualifies for Medicaid under the older-child category, which makes the child ineligible for CHIP."
+us,scenario_038,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model tested only whether about $22,993 of income was below a CHIP limit of about 250% FPL. It skipped the prior Medicaid determination: Child 2 (age 6) is Medicaid-eligible under the older-child category at this income, and that excludes the child from CHIP."
us,scenario_038,child2_head_start_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted a five-person household even though the listed household has only the head, spouse, and two children. It then applied that inflated household-size poverty threshold and treated age six as preschool age, producing eligibility despite the failed Head Start rules."
us,scenario_038,child2_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly recognized that Head Start preschool ordinarily serves ages three through five, then invented an unlisted exception by assuming the six-year-old had not entered kindergarten. The prompt requires unlisted statuses to be false, so that exception cannot establish eligibility."
us,scenario_038,child2_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model acknowledged that the child had aged out at six but nevertheless assumed the child had not entered kindergarten. No delayed-entry status was provided, and the prompt forbids inferring it, so the model improperly overrode the preschool-age restriction."
@@ -3296,52 +3397,54 @@ us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili
us,scenario_038,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old within the modeled WIC child age category. Because eligibility ends at the fifth birthday, Child 2 fails the categorical age test."
us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable and therefore failed the required submission contract.
us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly derived zero taxable income, zero tax before credits, and zero nonrefundable credits used, but its submitted value of -$1,644 contradicts that derivation. It inserted an unsupported negative amount even though nonrefundable credits cannot reduce liability below zero and refundable credits are excluded from this output."
-us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909 but then cut its own $7,428 EITC maximum to an arbitrary $7,239, submitting $7,239 + $2,909 = $10,148. The 2026 two-child EITC plateau maximum is $7,316."
-us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It gave only the 40% phase-in amount ($6,251), even though earnings of about $21,896 are on the plateau and earn the $7,316 maximum. It then subtracted an invented $1,656 EITC phase-out that does not apply below the MFJ phase-out start. It also granted a flat $3,400 ACTC without the 15% × (earned income − $2,500) limit, which yields $2,909."
-us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly derived the ACTC phase-in amount of $2,909 (below the $3,400 cap) but then submitted an unexplained $2,383. It also used an EITC of $7,150 instead of the 2026 two-child plateau maximum of $7,316."
-us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It placed the household 'near the plateau' and gave only a $5,500 EITC, although $21,896 of earnings is squarely on the plateau at $7,316. It computed an ACTC phase-in of about $3,074 below the $3,400 cap and then submitted $1,527 anyway. The correct ACTC is 15% × ($21,896 − $2,500) = $2,909."
-us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It estimated the EITC at about $7,000 instead of the $7,316 2026 two-child plateau maximum. It based the ACTC on gross earnings of $22,993 rather than earnings net of half the SE tax, and its $9,199 total implies an ACTC of about $2,199 instead of $2,909."
-us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909 but used a stale two-child EITC maximum of $6,604 as a proxy for 2026. The 2026 plateau maximum is $7,316, so the EITC was $712 too low."
-us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It projected the two-child EITC maximum at about $6,900 instead of $7,316, and it measured earned income as wages plus 92.35% of SE income ($21,806). It then cut its own $9,796 sum to an arbitrary $8,500 by citing a 'phase-in curve', although the household is on the EITC plateau."
-us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,430 instead of $7,316. It also measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax (about $21,896), which put the ACTC at $2,895.80 instead of $2,909.35."
-us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child CTC with a $3,000 threshold, capping the ACTC at $2,000. For 2026 the CTC is $2,200 with a $1,700 refundable cap, so the ACTC is 15% × (earned income − $2,500) = $2,909.35. It also set the EITC maximum at $7,354 instead of $7,316."
-us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It capped the ACTC at 2 × $1,000 = $2,000 using the reverted pre-TCJA credit and a $3,000 threshold. The 2026 law has a $1,700-per-child refundable cap and a $2,500 threshold, which gives $2,909.35. It also overstated the EITC maximum at $7,344 instead of $7,316."
-us,scenario_038,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a two-child EITC maximum of $7,350 instead of $7,316. It also measured ACTC earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax, which gave $2,895.80 instead of $2,909.35."
-us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the refundable CTC as the pre-TCJA $1,000 per child with a $3,000 threshold, capping it at $2,000. The 2026 refundable cap is $1,700 per child, so the 15% × ($21,896 − $2,500) = $2,909.35 phase-in binds. It also used a $7,300 EITC instead of $7,316."
-us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It counted the full $4,000 CTC as refundable, ignoring the $1,700-per-child refundable cap and the 15% earned-income phase-in that limits the ACTC to $2,909.35. It also gave an EITC of only $4,100, although earnings of about $21,896 are on the two-child plateau at $7,316."
-us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $2,000 CTC figure, which corresponds to the reverted $1,000-per-child credit. The 2026 refundable CTC is 15% × (earned income − $2,500) = $2,909.35, under a $3,400 cap. It also used a $7,300 EITC instead of $7,316."
-us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed an ACTC of $1,000 per child ($2,000) under the reverted pre-TCJA credit. The 2026 $1,700-per-child refundable cap makes the 15% earned-income phase-in ($2,909.35) binding. It also understated the EITC maximum at $7,250 instead of $7,316."
-us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Its $7,466 total is barely above the $7,316 EITC alone, which means it dropped nearly all of the $2,909.35 ACTC. That ACTC equals 15% of earned income above $2,500 for a family with two children and no income tax liability."
-us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $9,753.44 is $472 below the correct $7,316 EITC plus $2,909.35 ACTC; with the ACTC at about $2,909, that implies an EITC of about $6,844. So it understated the EITC below the $7,316 plateau maximum that applies at $21,896 of earnings."
-us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $10,185 is $40 below the correct $7,316 EITC plus $2,909.35 ACTC, which fits a slightly low 2026 two-child EITC maximum (about $7,276) paired with an otherwise correct ACTC phase-in."
-us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It capped the ACTC at $2,000 ($1,000 per child), as if the TCJA had lapsed. For 2026 the refundable cap is $1,700 per child, so the ACTC is the 15% × (earned income − $2,500) phase-in of $2,909.35. It also projected the EITC maximum at $7,326 instead of $7,316."
-us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to grade."
-us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $7,300 EITC instead of the $7,316 2026 two-child maximum. It also measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax (about $21,896), which understated the ACTC as $2,896 instead of $2,909.35."
-us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It gave no breakdown. Its $8,724 is $1,501 short of the correct $7,316 EITC plus $2,909.35 ACTC, so it understated one or both credits. The household is on the two-child EITC plateau, and its ACTC is the full 15% × (earned income − $2,500) phase-in amount."
-us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It said a family with two children and $22,993 of earnings generates no refundable credits. That family qualifies for the maximum two-child EITC ($7,316) and a $2,909.35 refundable ACTC."
-us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It had the EITC right at $7,316 but granted a flat $1,800-per-child refundable CTC ($3,600). It skipped the 15% × (earned income − $2,500) limit, which caps the ACTC at $2,909.35, below the $3,400 two-child refundable cap."
-us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $8,231, the three-or-more-child EITC maximum, for a two-child family whose maximum is $7,316. It also computed the ACTC on gross earnings of $22,993 without subtracting the deductible half of SE tax, which gave $3,073.95 instead of $2,909.35."
-us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $7,316, but its ACTC of $2,731.32 implies earned income of about $20,709. That figure comes from taking 92.35% of SE income and then also subtracting the half-SE-tax deduction, reducing SE earnings twice. The correct earned income is SE income minus half the SE tax plus wages (about $21,896), which gives $2,909.35."
-us,scenario_038,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It had the EITC right at $7,316 but granted the full $1,700-per-child refundable CTC ($3,400). It never applied the 15% × (earned income − $2,500) phase-in, which binds at $2,909.35."
-us,scenario_038,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It computed the ACTC phase-in correctly (about $2,909), but it overstated the 2026 two-child EITC maximum as $7,427 instead of $7,316."
-us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It misread the no-inference instruction as barring refundable credits and returned $0. The listed wages, SE income and two qualifying children directly produce a $7,316 EITC and a $2,909.35 refundable ACTC."
-us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had lapsed and capped the ACTC at $2,000 ($1,000 per child). In 2026 the CTC is $2,200 with a $1,700-per-child refundable cap, so the 15% × ($21,896 − $2,500) = $2,909.35 phase-in governs. It also uprated a stale 2024 EITC maximum to $7,312 instead of $7,316."
-us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a reverted $1,000-per-child ACTC with a $3,000 threshold, which gives $2,000. The 2026 refundable cap is $1,700 per child with a $2,500 threshold, so the ACTC is $2,909.35. It also projected the EITC maximum at $7,330 instead of $7,316."
-us,scenario_038,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a 'post-2025' CTC of $1,000 per child, as if the TCJA had expired, capping the refundable CTC at $2,000. The 2026 $1,700-per-child refundable cap lets the 15% earned-income phase-in ($2,909.35) bind. It also set the EITC maximum at $7,332 instead of $7,316."
-us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'post-TCJA reversion rules' ($1,000 per child, $3,000 threshold) to cap the ACTC at $2,000. For 2026 the ACTC is 15% × (earned income − $2,500) = $2,909.35 under a $3,400 cap. It also used gross earnings of $22,993 and an EITC maximum of $7,338 instead of $7,316."
-us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as about $7,422 instead of $7,316. It also understated the ACTC at $2,879 instead of 15% × ($21,896 − $2,500) = $2,909.35."
-us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to grade."
-us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $7,316 but computed the ACTC phase-in on gross earnings of $22,993. ACTC earned income counts SE income net of the deductible half of SE tax (about $21,896), which gives $2,909.35 instead of $3,073.95."
-us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It set the plateau EITC at about $6,500 instead of the $7,316 2026 two-child maximum. It then granted the full $3,400 refundable CTC without the 15% × (earned income − $2,500) limit, which caps the ACTC at $2,909.35."
-us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It computed the ACTC correctly at about $2,910 but overstated the 2026 two-child EITC plateau maximum as $7,370 instead of $7,316."
-us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It cycled through several parameter sets, including an invented $21,370 MFJ phase-out start that is below the true 2026 threshold. It then dropped its computed ACTC to $0 'since I'm uncertain' and submitted an arbitrary $6,868.75. The correct result is the $7,316 plateau EITC plus a $2,909.35 ACTC."
-us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed $22,993 of earnings exceeds the two-child EITC phase-out and gave only $46. MFJ earnings of about $21,896 are below the 2026 phase-out start, so the full $7,316 plateau credit applies. It also set the ACTC at an unsupported $3,000 instead of 15% × ($21,896 − $2,500) = $2,909.35."
+us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909, then arbitrarily cut the EITC from its $7,427 estimate to an unsupported $7,239. The 2026 two-child plateau maximum is $7,316, which gives a total of $10,225."
+us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It invented a $1,656 EITC phase-out even though earned income of about $21,896 is below the joint phase-out start of about $31,000, so the full $7,316 maximum applies. It also gave the full $3,400 ACTC cap without applying the 15%-of-earnings-above-$2,500 limit, which gives $2,909."
+us,scenario_038,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It assumed income tax fully absorbed the CTC, but income tax before credits is $0, so $2,909 is refundable through the 15% earned-income formula. It also gave far too low an EITC: earned income of $21,896 is on the plateau, so the EITC is the $7,316 maximum."
+us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an EITC of $7,150, below both the true 2026 maximum of $7,316 and its own stated $7,250 plateau. It then calculated the ACTC as $2,909 but entered an unexplained $2,383 in the total."
+us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It put the EITC at $5,500, as if earnings were still in the phase-in range, but $21,896 is past the roughly $18,290 phase-in end, so the EITC is the full $7,316. It also cut the ACTC to an arbitrary $1,527 even though the 15% formula gives $2,909, below the $3,400 cap."
+us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It underestimated the 2026 two-child EITC maximum as about $7,000 instead of $7,316. It also based the ACTC on gross earnings ($3,074) without subtracting half of the SE tax, and its $9,199 total doesn't match either component."
+us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It calculated the ACTC correctly at $2,909, but used $6,604 as the EITC maximum. That is the 2023 two-child figure, not the 2026 maximum of $7,316, which understates the total by $712."
+us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It estimated the EITC maximum at about $6,900 instead of $7,316. It then cut its own $9,796 sum to $8,500 with an unjustified 'phase-in' adjustment, even though earned income of $21,896 is on the plateau."
+us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,430 instead of $7,316. It also measured ACTC earnings as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which gave $2,895.80 instead of $2,909.35."
+us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child CTC with a $3,000 earnings floor, which capped the ACTC at $2,000. Under the 2026 law, the CTC is $2,200 per child, the refundable cap is $1,700 per child, and the floor is $2,500, which gives $2,909. It also overstated the EITC at $7,354 instead of $7,316."
+us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It capped the ACTC at $2,000, using $1,000 per child and a $3,000 floor. The 2026 rules have a $1,700-per-child refundable cap and a $2,500 floor, which gives $2,909. It also used $7,344 instead of the $7,316 EITC maximum."
+us,scenario_038,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used $7,350 for the EITC maximum instead of the 2026 figure of $7,316. It also based the ACTC on wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which gave $2,895.80 instead of $2,909.35."
+us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the refundable CTC as $1,000 per child with a $3,000 floor, totaling $2,000. Under the 2026 rules, the refundable CTC is up to $1,700 per child, phased in at 15% above $2,500, which gives $2,909. It also understated the EITC at $7,300 instead of $7,316."
+us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It treated the full $4,000 CTC as refundable, ignoring the $1,700-per-child refundable cap and the 15%-of-earnings-above-$2,500 limit, which gives $2,909. It also put the EITC at $4,100, even though earned income of $21,896 is on the plateau, where the credit is the full $7,316."
+us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $2,000 CTC, reflecting the pre-TCJA $1,000 per child. The 2026 refundable CTC formula gives $2,909: 15% of $19,396, below the $3,400 cap. It also put the EITC at $7,300 instead of $7,316."
+us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It capped the ACTC at $1,000 per child ($2,000 total) under the pre-TCJA rules. In 2026 the refundable cap is $1,700 per child, so the 15% formula's $2,909 applies. It also used $7,250 instead of the $7,316 EITC maximum."
+us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $7,466 total is about $2,759 below the correct figure. That matches dropping most of one credit, such as using the EITC plus only a token ACTC. The correct components are the $7,316 plateau-maximum EITC and a $2,909 ACTC (15% of the $19,396 of earnings above $2,500)."
+us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $9,753.44 total is about $472 short. That matches an understated EITC maximum or ACTC earnings base. The correct components are the $7,316 two-child EITC maximum, since earned income of $21,896 is on the plateau, and a $2,909.35 ACTC."
+us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $10,185 total is $40 short of $7,316 EITC plus $2,909.35 ACTC. That matches using a projected EITC maximum near $7,276 instead of the published 2026 figure of $7,316."
+us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It set the ACTC at $2,000, using the pre-TCJA $1,000 per child, instead of applying the 2026 cap of $1,700 per child to the 15% formula, which gives $2,909. It also used $7,326 instead of the $7,316 EITC maximum."
+us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no parseable value for federal_refundable_credits, so there was no answer to score against $10,225.35."
+us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which understated the ACTC at $2,896. It also rounded the EITC to $7,300 instead of the 2026 maximum of $7,316."
+us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $8,724 total is about $1,501 short. That matches an EITC well below the plateau maximum or a pre-TCJA $2,000 ACTC. The correct EITC is the $7,316 maximum, since earned income of $21,896 is on the plateau, and the ACTC is $2,909 from 15% of earnings above $2,500."
+us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It reported no refundable credits, even though the joint filers have $21,896 of earned income and two qualifying children. That qualifies them for the full $7,316 EITC and a $2,909 refundable CTC."
+us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Its EITC of $7,316 is correct, but it gave a flat $1,800 per child ($3,600) of refundable CTC. It ignored the earned-income phase-in, which limits the refundable CTC to 15% × ($21,896 − $2,500) = $2,909, below the $1,700-per-child cap."
+us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $8,231, the 2026 EITC maximum for three or more children, instead of the $7,316 two-child maximum. It also based the ACTC on gross earnings of $22,993 without subtracting half the SE tax, which gave $3,073.95 instead of $2,909.35."
+us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its EITC of $7,316 is correct, but its $2,731.32 ACTC implies earned income of $20,709. That matches applying the 92.35% factor to SE income and then also subtracting half the SE tax. Net SE earnings are $15,525 minus $1,097 only, so earned income is $21,896 and the ACTC is $2,909.35."
+us,scenario_038,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its EITC of $7,316 is correct, but it gave the full $1,700-per-child refundable cap ($3,400) without the earned-income limit. The refundable CTC is limited to 15% × ($21,896 − $2,500) = $2,909.35."
+us,scenario_038,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its $2,909.46 ACTC is essentially correct, but it used $7,427 as the two-child EITC maximum instead of the published 2026 figure of $7,316, which overstates the total by $111."
+us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It wrongly treated the 'do not infer unlisted facts' instruction as a bar on computing credits and returned $0. The listed wages, SE income and two children directly produce a $7,316 EITC and a $2,909 refundable CTC."
+us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA lapsed, so it used a $1,000 CTC per child and an ACTC of $2,000. For 2026, the refundable cap is $1,700 per child, so the 15% formula's $2,909 applies. It also uprated the EITC maximum to $7,312 instead of the published $7,316."
+us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child ACTC with a $3,000 floor ($2,000 total) instead of the 2026 cap of $1,700 per child and the $2,500 floor, which give $2,909. It also projected a $7,330 EITC maximum instead of $7,316."
+us,scenario_038,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed the CTC reverted to $1,000 per child after 2025, giving a $2,000 refundable CTC. The 2026 law keeps a $2,200 CTC with a $1,700 refundable cap, so the 15% formula gives $2,909. It also used $7,332 instead of the $7,316 EITC maximum."
+us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied post-TCJA reversion rules ($1,000 per child, $3,000 floor) for an ACTC of $2,000, instead of the 2026 cap of $1,700 per child and the $2,500 floor, which give $2,909. It also projected the EITC at $7,338 instead of $7,316."
+us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,422 instead of $7,316. Its ACTC of $2,879 is below the correct $2,909.35, which is 15% × ($21,896 − $2,500)."
+us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no parseable value for federal_refundable_credits, so there was no answer to score against $10,225.35."
+us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $7,316 is correct, but it based the ACTC on gross earnings of $22,993. Earned income for the CTC phase-in counts SE income net of half the SE tax ($21,896), so the ACTC is $2,909.35, not $3,073.95."
+us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It understated the EITC maximum at $6,500 instead of the 2026 plateau amount of $7,316. It also gave the full $3,400 ACTC cap without the 15%-above-$2,500 limit, which caps it at $2,909."
+us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its $2,910 ACTC is correct, but it used $7,370 as the 2026 two-child EITC maximum instead of $7,316, which overstates the total by about $55."
+us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It wrongly applied a $21,370 joint EITC phase-out start; in 2026 the phase-out starts around $31,000, so $21,896 of earnings is on the plateau and earns $7,316. It then abandoned its own ACTC calculation of $2,909 and submitted an unsupported EITC-only figure of $6,868.75."
+us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed earned income of $22,993 was past the EITC phase-out and gave only $46. For joint filers with two children, the 2026 phase-out starts around $31,000, so the full $7,316 applies. It also rounded the ACTC to $3,000 instead of $2,909.35, which uses earnings net of half the SE tax."
us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted nonqualification without applying either qualifying pathway. PolicyEngine derives categorical eligibility from the household's $7,286.94 annual SNAP benefit and independently derives income eligibility because the school-meal FPG ratio is 0.70, below 1.30."
us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly computed gross income below 130% FPG but then incorrectly imposed SNAP receipt or direct certification as an additional requirement and treated an unlisted SNAP input as proof of no SNAP participation. Income at or below 130% FPG independently qualifies the children for free meals, and PolicyEngine also computes $7,286.94 of SNAP under the assumed-take-up instruction, which supplies categorical eligibility and produces positive meal support."
us,scenario_038,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_038,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,health_coverage,False,"The model applied Louisiana's pre-expansion parent/caretaker income standard (roughly 24% FPL) rather than the ACA adult expansion limit of 138% FPL that Louisiana has used since July 1, 2016 for the MAGI ADULT category. Combined wage and self-employment MAGI of about $22,992 for the four-person unit is 0.66 FPL, less than half the 138% ceiling, so the 25-year-old head qualifies on income alone."
-us,scenario_038,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,health_coverage,False,"The model declared the head over the limit under both non-expansion and expansion criteria without ever computing the income-to-poverty ratio; $7,468 in wages plus $15,525 in self-employment income for a household of four is 0.66 FPL. Louisiana's expansion pathway covers non-disabled adults under 65 up to 138% FPL, so 66% FPL clears the standard by a wide margin and the head is eligible in the MAGI ADULT category."
-us,scenario_038,head_medicaid_eligible,gpt-5.4-nano,llm_error,health_coverage,False,"The model gated eligibility on 'income/assets,' but MAGI-based Medicaid for ACA expansion adults imposes no resource test, so the $500 bank balance and $2,090 vehicle are irrelevant and the sole test is MAGI against 138% FPL, which the head passes at 0.66 FPL. Its zero is consistent with treating Louisiana as a non-expansion state and dropping a 25-year-old working parent at 66% FPL into the coverage gap, when Louisiana expanded in 2016."
+us,scenario_038,head_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said the head's income was above Louisiana's adult expansion limit, but it never computed household MAGI against the FPL for a household of four. About $22,992 in combined wage and self-employment income is 0.66x FPL, well under the 138% FPL limit for ACA expansion adults, so the head qualifies in the ADULT category."
+us,scenario_038,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said the head's income was above the Louisiana threshold for parents and adults. It applied a parent/caretaker cutoff as if Louisiana had not expanded, instead of the 138% FPL ACA expansion limit for the ADULT category. The head's MAGI of about $22,992 for a four-person household is 0.66x FPL, which meets that limit."
+us,scenario_038,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model claimed the head fails both the expansion and non-expansion criteria, but it never compared the roughly $22,992 household MAGI to the 4-person FPL. That income is 0.66x FPL, under Louisiana's 138% FPL limit for expansion adults, so the head is eligible through the ADULT category."
+us,scenario_038,head_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model cited the household's income and assets to deny eligibility. MAGI-based expansion Medicaid has no asset test, and the head's household MAGI of about $22,992 is 0.66x FPL for a family of four, below Louisiana's 138% FPL limit for ACA expansion adults. The model's 0 means it skipped the expansion income test."
us,scenario_038,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a WIC categorical pathway for a parent or guardian of children. Only pregnant, postpartum, or breastfeeding women, infants, and children under age 5 qualify categorically; the head has no qualifying status and both children are over age 5."
us,scenario_038,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated parent or caretaker status as WIC categorical eligibility and also misstated the ages 7 and 6 children as under 5. Low income and adjunctive eligibility through SNAP or Medicaid satisfy an income pathway only; they do not replace the required pregnant, postpartum, breastfeeding, infant, or under-5 category."
us,scenario_038,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model converted reproductive age into pregnant or postpartum status despite the prompt directing that unlisted statuses are false. A 25-year-old adult is not WIC-eligible without an actual pregnant, postpartum, or breastfeeding status, and assumed program take-up does not establish categorical eligibility."
@@ -3352,52 +3455,54 @@ us,scenario_038,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals
us,scenario_038,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model’s submitted $569.90 contradicts its own stated component sum of $571.30, so it failed to carry its calculation into the output. It also recomputed Social Security from the displayed rounded wage figure instead of using the engine’s component result of $462.98; adding that to $108.28 gives $571.26."
us,scenario_038,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model treated 185% of the federal poverty guideline as a one-sided reduced-price eligibility ceiling and failed to apply the free-tier test first. The household is at 70% of the guideline and categorically eligible, so it qualifies for free meals and cannot receive reduced-price support."
us,scenario_038,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model used incorrect 130% and 185% poverty thresholds, causing it to place $22,993 inside the reduced-price band even though the engine's ratio is 70%. It also inferred no categorical eligibility from the prompt despite assumed program take-up; the household meets categorical eligibility and receives the mutually exclusive free-meals tier."
-us,scenario_038,self_employment_tax,claude-fable-5,llm_error,other,False,"The model derived the correct base and rate — $15,525 × 0.9235 = $14,337.34, × 0.153 ≈ $2,193.61 — and stated $2,193.61 in its own explanation, then submitted $2,196.28, a figure $2.67 higher that no step of its derivation produces. The failure is transcription of the final answer, not the SECA rule: its stated computation is exact and matches the reference to the cent."
-us,scenario_038,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model applied the correct 92.35%/15.3% structure but botched both multiplications: $15,525 × 0.9235 is $14,337.34, not the $14,339.59 it wrote, and $14,339.59 × 0.153 is $2,193.96, not the $2,194.96 it submitted. The $1.35 overstatement of self-employment tax is pure arithmetic drift in the net-earnings base and the rate application, with no rule misapplied."
-us,scenario_038,self_employment_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model asserted $2,191.73 with no shown work; that figure equals 15.3% of a net-earnings base of $14,325.03, which is $12.31 below the correct 92.35% base of $14,337.34 on $15,525 of self-employment income. It understated the SECA base by mis-multiplying the 0.9235 reduction rather than by applying a wrong rate — the 15.3% combined rate it implicitly used is correct because wages plus net earnings fall far below the Social Security wage base."
-us,scenario_038,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the standard SE tax computation but reported a whole-dollar estimate of $2,197 without executing it; that value corresponds to 15.3% of a $14,359.48 base, $22.14 above the correct 92.35% net-earnings base of $14,337.34. Running the formula it cited on $15,525 yields $2,193.61, so the $3.39 error is an unshown approximation substituted for the arithmetic."
-us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated the correct method — 15.3% of 92.35% of $15,525 — which yields $2,193.61, then reported $1,360.68, a figure equal to 15.3% of only $8,893.99, about 62% of the correct $14,337.34 net-earnings base. It never carried its own stated base through the rate multiplication, producing a number inconsistent with every step it described."
-us,scenario_038,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model explicitly settled for an approximation of the 15.3% SECA rate on net earnings and returned a whole-dollar $2,192, which corresponds to a base of $14,326.80 rather than the exact 92.35% base of $14,337.34 on $15,525. Carrying the same formula to the cent gives $2,193.61; the $1.61 shortfall is rounding the base down instead of computing it."
-us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for self_employment_tax, so the submission carries no substantive position on the computation. The required output — 15.3% × 0.9235 × $15,525 = $2,193.61 — was simply absent from the response contract."
-us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model computed the correct answer — $15,525 × 0.9235 = $14,337.34, × 0.153 = $2,193.61 — then subtracted a second adjustment for the $1,096.81 one-half-of-SE-tax deduction, claiming it feeds back into the SECA base through an iterative computation. The §164(f) deduction reduces adjusted gross income for income tax purposes only and never re-enters the self-employment tax base; the 92.35% factor is the sole adjustment for the employer-share equivalent, so applying it twice drove the answer $139.53 below the correct $2,193.61."
-us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its reasoning worked out about $7,236 a year: a $1,000 guessed maximum minus 30% of $1,321 net, using a $212 standard deduction. It then submitted $4,200, which its own work does not support. Even the computed figure used the wrong FY2026 parameters; the correct ones are a $223 standard deduction and a $994 maximum, giving $601 a month."
-us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never applied the 20% earned income deduction and used a $194 standard deduction. It also invented a $400/month shelter deduction from the $75,000 mortgage balance, although no mortgage payment, property tax or utility cost is listed. It then dropped to an unexplained $84/month. The correct result is $1,916 − $383 − $223 = $1,310 net, and $994 − $393 = $601/month."
-us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It took 20% off annual earnings ($18,394) and then subtracted a monthly standard deduction (~$224) from that annual figure. That shrank the standard deduction to about $19 a month and gave net income of $1,514 instead of $1,310. It then abandoned its own $540/month result for an unsupported $759/month ($9,114), well above the correct $601/month."
-us,scenario_038,snap,claude-opus-4.8,llm_error,other,False,"It named the right inputs ($1,916 gross, 20% earned income deduction, ~$217 standard deduction, $994 maximum) but never did the arithmetic. Those inputs give about $599/month. Its $7,848 ($654/month) implies a 30% contribution of only ~$340, which none of its stated deductions support. The correct result is $994 − $393 = $601/month."
-us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"It described the right method (20% earned income deduction, standard deduction, 30% contribution) but produced $564/month without showing parameters, $37 below the correct $601. The shortfall comes from not using the FY2026 $994 maximum allotment and $223 standard deduction, which give net income of $1,310 and a $393 contribution."
-us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the FY2025 figures of a $975 maximum allotment and a $204 standard deduction instead of FY2026's $994 and $223. That raised net income to $1,328.86 and gave $576/month. The FY2026 parameters give $1,310 net, a $393 contribution and $601/month."
-us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It cut self-employment income by 20% before counting it, which understated gross income at $1,657/month. It then took the 20% earned income deduction a second time and added dependent care and medical deductions, even though there are no care costs and no elderly or disabled members. The resulting net of $900-$1,000 was also discarded in favour of an unsupported $6,800. The correct net income is $1,310, giving $601/month."
-us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated gross income at $21,896/year instead of $22,993. It also used an annual maximum allotment of $13,164 ($1,097/month) instead of $994 × 12 = $11,928, which inflated the benefit to $8,697. The correct figures are $1,310 net and $601/month."
-us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and a $208 standard deduction instead of FY2026's $994 and $223. It also left the contribution and benefit unrounded, giving $652.54/month instead of $994 − $393 = $601."
-us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 $975 maximum allotment and $204 standard deduction instead of FY2026's $994 and $223. That gave net income of $1,328.87 and a benefit of $576.34/month, below the correct $601."
-us,scenario_038,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $213 standard deduction and a $1,000 maximum allotment instead of $223 and $994. It also skipped rounding net income to $1,310 and the contribution to $393, which produced $604.04/month instead of $601."
-us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It made up a $310/month 'dependent deduction'. SNAP has no per-dependent deduction, and the dependent care deduction needs actual care costs, which are $0 here. It also used the 2024 $973 maximum and $198 standard deduction, which pushed net income down to $1,025 and the benefit up to $665.50/month instead of $601."
-us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave a round $9,000 ($750/month) without doing any calculation. The correct steps are $1,916.04 − $383.21 earned income deduction − $223 standard deduction = $1,310 net, then $994 − $393 = $601/month. The $750 figure overstates that by $149 a month."
-us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It stated the right formula (maximum allotment minus 30% of net income) but produced $573/month. That matches pre-FY2026 parameters (about $975 maximum and $204 standard deduction), not FY2026's $994 and $223, which give $601/month."
-us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $1,333 net income means it used a standard deduction of about $200 instead of FY2026's $223. It also used a ~$990 maximum instead of $994, which gave $590/month instead of $994 − $393 = $601."
-us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It concluded the household's net income rules out any benefit. In fact, net income of $1,310 is only 48-49% of the poverty guideline, far under the 100% net income limit, and gross income is 70-72% of the guideline. The benefit is $994 − $393 = $601/month."
-us,scenario_038,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It said the household passes all tests but gave $523/month. That matches a 30% contribution on about $1,507-$1,570 of net income, which is what you get by dropping the $223 standard deduction. The correct net of $1,310 gives a $393 contribution and $601/month."
-us,scenario_038,snap,gemini-3.7-flash,llm_error,other,False,"It gave $809/month ($9,708) without any calculation. Against the $994 maximum, that implies net income of only about $617, roughly half the correct $1,310 ($1,916.04 − $383.21 − $223). No deduction beyond the earned income and standard deductions applies, so the correct benefit is $601/month."
-us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the right structure (20% earned income deduction plus standard deduction, then 30% of net off the maximum) but reached $580.50/month. That matches pre-FY2026 parameters instead of the $994 maximum and $223 standard deduction, which give $601/month."
-us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It estimated the standard deduction at $220 and the maximum allotment at $1,020 instead of FY2026's $223 and $994, and left the result unrounded. That gave $626.14/month instead of $994 − $393 = $601."
-us,scenario_038,snap,glm-5.3,llm_error,other,False,"It reached the correct $1,310 net income but took 30% of it as $394 instead of $393 and used a $995 maximum. It then rounded the benefit to 'about $600/month' instead of computing $994 − $393 = $601, which left the annual figure $12 short at $7,200."
-us,scenario_038,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It declared a zero benefit because of earnings and assets. But gross income is 70-72% of the poverty guideline, net income is 48-49%, and the $500 of assets is well under the limit; TANF non-cash categorical eligibility also applies. The formula gives $994 − 30% × $1,310 = $601/month."
-us,scenario_038,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It asserted a zero benefit without applying any test. The household passes the gross, net and asset tests, with net income of $1,310 after the 20% earned income deduction and the $223 standard deduction. That gives $994 − $393 = $601/month."
-us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"Its net income of about $1,316 was close to right, but it used a maximum allotment of $879 for four people instead of FY2026's $994. That cut the benefit to $484/month instead of $601."
-us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It described the right formula but produced an unrounded $609.74/month. That implies an overstated maximum allotment or understated contribution compared with the FY2026 $994 maximum and $393 contribution (30% of $1,310). It also skipped SNAP's whole-dollar rounding."
-us,scenario_038,snap,gpt-6-luna,llm_error,thresholds_rates,False,"It got $595/month, which implies a $399 contribution on net income of about $1,330. That is the result of using the FY2025 $204 standard deduction instead of FY2026's $223, which gives $1,310 net, a $393 contribution and $601/month."
-us,scenario_038,snap,grok-4.3,llm_error,other,False,"It entered 0 because program parameters 'cannot be inferred', misreading the no-inference rule. That rule covers unlisted household facts, not statutory SNAP parameters. With the listed earnings, the FY2026 formula gives $601/month."
-us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a ~$225 standard deduction and a ~$1,000 maximum allotment instead of $223 and $994, which gave $608/month. It also mis-annualized: $608 × 12 is $7,296, not the $7,280 it submitted. The correct figure is $601/month."
-us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction and reached $1,310 net. But it used a $993 maximum allotment instead of FY2026's $994, which gave $600/month instead of $994 − $393 = $601."
-us,scenario_038,snap,grok-4.7,llm_error,thresholds_rates,False,"It used an $11,700 annual maximum allotment ($975/month, the FY2025 figure) and a $2,604 annual standard deduction ($217/month). FY2026's $994 maximum and $223 standard deduction are needed to get $1,310 net and $601/month."
-us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a ~$3,985 annual standard deduction ($332/month) and a ~$12,337 annual maximum allotment ($1,028/month). Both overstate FY2026's $223 standard deduction and $994 maximum and push the benefit to $8,014. The correct result is $601/month."
-us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"Its net income of about $1,316 reflects a standard deduction of roughly $217 instead of $223. It also used a ~$1,000 maximum allotment instead of $994, which gave $605/month instead of $601."
-us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $208 standard deduction and a $1,020 maximum allotment instead of FY2026's $223 and $994. That gave net income of $1,325 and $622/month instead of $1,310 net and $601/month."
-us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction but used a $997 maximum allotment instead of $994. It also skipped rounding net income to $1,310 and the contribution to $393, which gave $604.04/month instead of $601."
-us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"Its $1,329 net income reflects the FY2025 $204 standard deduction instead of $223, and it used a ~$1,050 maximum allotment instead of FY2026's $994. That inflated the benefit to $651/month instead of $601."
-us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It used the correct $994 maximum but a ~$207 standard deduction instead of $223. That raised net income to $1,326 and the contribution to $398, giving about $596/month instead of $994 − $393 = $601."
-us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It first computed $572.54/month using an outdated $198 standard deduction and $973 maximum. It then made up a shelter deduction to raise the benefit to $879/month, although no rent, mortgage payment or utility cost is listed, so no excess shelter deduction applies. The correct figures are $1,310 net and $601/month."
-us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income to $0 with a shelter deduction the household cannot claim, because no rent, mortgage payment or utility cost is listed. It also used a $964 maximum instead of $994. The standard and earned income deductions leave $1,310 of net income, so the $393 contribution applies and the benefit is $601/month, not the maximum."
+us,scenario_038,self_employment_tax,claude-fable-5,llm_error,other,False,"Its own reasoning reached the correct $14,337.34 × 0.153 = $2,193.61, but it submitted 2196.28, which does not match its computation. This is a transcription error between the derivation and the final value."
+us,scenario_038,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"It miscalculated $15,525 × 0.9235 as $14,339.59 instead of $14,337.34. Applying 15.3% to that inflated base gave $2,194.96 instead of $2,193.61."
+us,scenario_038,self_employment_tax,claude-haiku-5.5,llm_error,other,False,"It stated the right method (92.35% of $15,525 at 15.3%) but made an arithmetic error. The product is $14,337.34 × 0.153 = $2,193.61, not $2,191.80."
+us,scenario_038,self_employment_tax,gemini-3.5-flash-lite,llm_error,other,False,"It gave $2,191.73 without showing any work. That is an arithmetic slip, because the standard computation $15,525 × 0.9235 × 0.153 gives $2,193.61."
+us,scenario_038,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"It cited the standard SE tax computation but reported $2,197, which is $3.39 too high. Correctly applied, 15.3% on 92.35% of $15,525 gives $2,193.61, so its number comes from imprecise arithmetic or rounding."
+us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It described the right method (15.3% of 92.35% of $15,525) but reported $1,360.68. That equals an effective rate of only about 9.5% on the $14,337.34 of net SE earnings, so it never actually applied the full 15.3% rate to the full base and badly understated the $2,193.61 liability."
+us,scenario_038,self_employment_tax,grok-4.3,llm_error,other,False,"It labeled its answer approximate and gave $2,192 instead of computing exactly. The exact figure is $15,525 × 0.9235 × 0.153 = $2,193.61."
+us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for self_employment_tax, so there was nothing to score. The correct result is $15,525 × 0.9235 × 0.153 = $2,193.61."
+us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It correctly reached $2,193.61, then applied the deduction for half of SE tax ($1,096.81) a second time, iteratively reducing the SE tax base itself and ending at $2,054.08. Under IRC §164(f) that deduction reduces only AGI for income tax. The 92.35% factor already accounts for it, so the SE tax stays at $2,193.61."
+us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its own working reached about $7,236 per year, but it submitted 4,200, a number its reasoning does not support. The working also used a $212 standard deduction and a $1,000 maximum allotment instead of the FY2026 values of $223 and $994, which give $601 a month."
+us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It skipped the 20% earned income deduction and used a $194 standard deduction instead of $223. It also invented a $400 shelter deduction from the mortgage balance, even though no housing payment is listed. It then dropped its own $647/month result for an unexplained $84/month, far below the correct $601 ($994 − 30% × $1,310)."
+us,scenario_038,snap,claude-haiku-5.5,llm_error,other,False,"It showed no calculation. Its $3,600 equals $300 a month, about half of the correct $601: $994 maximum allotment minus 30% of $1,310 net income, after the $223 standard deduction and the $383 earned income deduction."
+us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted a monthly standard deduction ($224) from annual income ($18,394), which inflated monthly net income to $1,514 instead of $1,310. It then dropped its own $540/month result and submitted an unsupported $759/month ($9,114/yr)."
+us,scenario_038,snap,claude-opus-4.8,llm_error,other,False,"With its own inputs ($994 maximum, $217 standard deduction, 20% earned income deduction), net income is about $1,316 and the benefit is about $599 a month. It never did that arithmetic and asserted $654/month ($7,848/yr) instead. The correct figure with the $223 deduction is $601."
+us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"It gave no parameters and asserted $564 a month. That is $37 below the correct $601, which comes from the $994 FY2026 maximum allotment minus 30% of $1,310 net income after the $223 standard deduction. Its figure reflects an understated allotment or overstated net income."
+us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deliberately used FY2025 parameters, a $975 maximum allotment and a $204 standard deduction, for a 2026 household. The FY2026 values are $994 and $223, which give net income of $1,310 and a $601 monthly benefit instead of its $576."
+us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It cut self-employment income by 20% before counting it, then applied the 20% earned income deduction to the reduced amount. It also assumed dependent care and medical deductions, although there are no care costs and no elderly or disabled members, which understated net income at $900–1,000. Even with that net income, its $994-based arithmetic doesn't produce the $570–600 a month it reported."
+us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated gross income at $21,896 instead of $22,993 (wages plus self-employment income). It used an annual maximum allotment of $13,164, which is $1,097 a month instead of $994, and a $219 standard deduction instead of $223, which inflated the benefit to $8,697."
+us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and a $208 standard deduction instead of the FY2026 values of $994 and $223. That overstated the monthly benefit at $652.54 instead of $601."
+us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied FY2025 parameters, a $975 maximum allotment and a $204 standard deduction, instead of the FY2026 values of $994 and $223. That gives $576.34 a month instead of $601."
+us,scenario_038,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $1,000 maximum allotment and a $213 standard deduction instead of $994 and $223, and it didn't round net income or the allotment to whole dollars. The result was $604.04 a month instead of $601."
+us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a $310/month 'dependent deduction'. SNAP has no per-dependent deduction, and the dependent care deduction requires actual care costs, which this household doesn't have. It also used the FY2024 $973 maximum allotment and a $198 standard deduction instead of $994 and $223."
+us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It did no calculation. Its $9,000 ($750/month) overstates the benefit, which is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month."
+us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It described the right formula but showed no numbers. Its $6,878 (about $573/month) understates the correct $601, which comes from the $994 FY2026 maximum allotment minus $393 (30% of $1,310 net income after the $223 standard deduction)."
+us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $990 maximum allotment instead of $994. Its net income of $1,333 implies a standard deduction of about $200 instead of the FY2026 $223, so its benefit came out at $590 a month instead of $601."
+us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It said the household's net income disqualifies it, but net income of $1,310 is only 49% of the poverty guideline. The household passes every test and is categorically eligible through TANF non-cash (broad-based categorical) eligibility, so it receives $601 a month."
+us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly found the household eligible but gave $6,276 ($523/month) with no calculation. The correct benefit is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month."
+us,scenario_038,snap,gemini-3.7-flash,llm_error,other,False,"It showed no calculation. Its $9,708 ($809/month) overstates the benefit, which is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month."
+us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It cited the right formula but no parameters. Its $580.50/month matches FY2025 values (around a $975 allotment) rather than the FY2026 $994 allotment and $223 standard deduction, which give $601."
+us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $1,020 maximum allotment instead of the FY2026 $994 and a $220 standard deduction instead of $223. That overstated the benefit at $626.14 a month instead of $601."
+us,scenario_038,snap,glm-5.3,llm_error,other,False,"It reached the correct net income of $1,310 but took 30% of it as $394 instead of $393 and used a $995 allotment. It then rounded to 'about $600/month' instead of computing $994 − $393 = $601, which is why it submitted $7,200 instead of $7,212."
+us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It said earnings and assets rule out a benefit. In fact, gross income is 72% of the poverty guideline, assets are $500, and the household is categorically eligible through TANF non-cash eligibility, so it receives $994 − $393 = $601 a month."
+us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying the income tests. Gross income is 72% and net income 49% of the poverty guideline, and the household has categorical eligibility, so the benefit is $601 a month."
+us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"It used a four-person maximum allotment of $879 instead of the FY2026 $994. Its net income of $1,316 implies a standard deduction of about $217 instead of $223, so it got $484 a month instead of $601."
+us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $609.74/month implies a maximum allotment of about $1,003 or net income below $1,310, and it never rounded to whole dollars. The FY2026 $994 maximum minus $393 gives exactly $601."
+us,scenario_038,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $595/month matches the $994 maximum allotment combined with the FY2025 $204 standard deduction (net about $1,329, 30% about $399). The FY2026 $223 standard deduction gives net income of $1,310 and a $601 benefit."
+us,scenario_038,snap,grok-4.3,llm_error,missing_output,False,"It refused to estimate and entered $0, wrongly treating the absence of program parameters as a reason to report no benefit. The listed facts are enough to compute $601 a month ($994 maximum minus 30% of $1,310 net income)."
+us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a projected $1,000 maximum allotment instead of $994 and a $225 standard deduction instead of $223, giving $608 a month. It then annualized that inconsistently as $7,280, while the correct figure is $601 × 12 = $7,212."
+us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction and $1,310 net income, but it used a $993 maximum allotment instead of the FY2026 $994. That made the benefit $600 a month instead of $601."
+us,scenario_038,snap,grok-4.7,llm_error,thresholds_rates,False,"It used an $11,700 annual maximum allotment ($975/month, the FY2025 value) and a $217/month standard deduction ($2,604/yr) instead of the FY2026 values of $994 and $223. That understated the annual benefit at $6,963."
+us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,985 annual standard deduction (about $332/month, far above $223) and a $12,337 annual maximum allotment (about $1,028/month instead of $994). Both errors push the benefit up, to $8,014."
+us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"It used a $1,000 maximum allotment instead of $994. Its net income of $1,316 implies a standard deduction of about $217 instead of $223, which gave $605 a month instead of $601."
+us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $208 standard deduction and a $1,020 maximum allotment instead of the FY2026 values of $223 and $994. That overstated the benefit at $622 a month instead of $601."
+us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction but used a $997 maximum allotment instead of $994. It also didn't round net income to $1,310 or the allotment to whole dollars, so it got $604.04 a month instead of $601."
+us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and about a $204 standard deduction (net $1,329) instead of the FY2026 values of $994 and $223. That overstated the benefit at about $651 a month instead of $601."
+us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It correctly used the $994 maximum allotment but used a $207 standard deduction instead of the FY2026 four-person value of $223. That made net income $1,326 instead of $1,310 and the benefit about $596 instead of $601."
+us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"With a $198 standard deduction and a $973 maximum allotment, it got $572.54 a month. It then invented a shelter deduction to raise that to $879, even though the household has no rent, mortgage payment or utility costs."
+us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income to $0 by assuming a shelter deduction, even though no housing costs are listed. It also skipped the 30% contribution on $1,310 net income and used a $964 maximum allotment instead of $994."
us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only the CHIP age and income conditions and skipped the prerequisite that the person not qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP."
us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the question asking about eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP even though the spouse is under 19 and below the CHIP income limit."
us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model concluded that satisfying the CHIP age and income ceiling was sufficient and never evaluated Medicaid eligibility first. The spouse is Medicaid-eligible under the OLDER_CHILD category, so the mutually exclusive CHIP pathway yields No."
@@ -3420,160 +3525,157 @@ us,scenario_038,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,s
us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It invented a $4,500 per-person exemption for all four members, totaling $18,000, and a 1.75% 'post-2024' bottom rate. The post-2024 reform actually set a flat 3% rate with a $25,000 MFJ standard deduction. That deduction exceeds the $21,896 AGI, so tax is $0 rather than $68."
us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits. The correct derivation is AGI of about $21,896 minus Louisiana's $25,000 MFJ standard deduction, which gives zero taxable income and $0 tax."
us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used the pre-reform $4,500 per-spouse exemptions and $1,000 dependent exemptions, then taxed $10,896 at the pre-2022 2%/4% bracket rates with a wrong $10,000 bracket break. It never applied the 2025 reform's $25,000 MFJ standard deduction and flat 3% rate, which bring taxable income to zero on AGI of $21,896."
-us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly placed the household at the federal EITC maximum and applied Louisiana's 5% rate. However, it used $7,478 as the 2026 two-child maximum instead of $7,316, which gave $374 rather than $365.80."
-us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Louisiana has no refundable EITC and only nonrefundable credits. It missed Louisiana's refundable EITC of 5% of the federal EITC, which is 0.05 x $7,316 = $365.80 for this household."
-us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model applied the correct 5% Louisiana rate but put the federal EITC at about $3,680. Joint filers with two children and about $21,900 of earned income are on the plateau below the MFJ phase-out, so the federal EITC is the full $7,316 and the LA credit is $365.80."
-us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model said earned income of about $21,900 was near the plateau, then used a federal EITC of only $2,640, which contradicts its own placement. On the plateau the federal EITC is the full two-child maximum of $7,316, so 5% gives $365.80 rather than $132."
-us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied Louisiana's 5% refundable EITC rate, but its $350 answer implies a federal EITC of about $7,000. The 2026 two-child maximum is $7,316, which gives $365.80."
-us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said Louisiana eliminated its refundable EITC. Louisiana still pays a refundable EITC of 5% of the federal credit, which is $365.80 on this household's $7,316 federal EITC."
-us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable. The credit is refundable at 5% of the federal EITC, so this household receives $365.80 regardless of its state tax liability."
-us,scenario_038,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model identified the federal EITC correctly as $7,316 but applied a 50% state match instead of Louisiana's 5% rate. That multiplied the correct $365.80 by ten."
-us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied a 3.5% state rate instead of Louisiana's current 5%. It also used a federal EITC of $7,430 instead of the 2026 two-child maximum of $7,316. The correct result is 0.05 x $7,316 = $365.80."
-us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but used $7,354 as the federal EITC instead of the 2026 two-child maximum of $7,316. That gave $367.70 instead of $365.80."
-us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no Louisiana refundable credit applies and missed the refundable LA EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80."
-us,scenario_038,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said Louisiana eliminated its state EITC. Louisiana still pays a refundable EITC of 5% of the federal credit, which is $365.80 here."
-us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits apply, overlooking the refundable LA EITC. At 5% of the household's $7,316 federal EITC, that credit is $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but used $7,250 as the federal EITC instead of the 2026 two-child maximum of $7,316. That gave $362.50 instead of $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave Louisiana refundable credits as zero, omitting the refundable LA EITC. That credit is 5% of the $7,316 federal EITC, or $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model identified Louisiana's refundable EITC, but its $353.43 answer is 5% of about $7,069. That understates the 2026 two-child federal maximum of $7,316, which is where this household sits on the plateau."
-us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model identified the Louisiana refundable EITC, but its $350 answer is 5% of a rounded federal EITC of $7,000. The actual 2026 two-child maximum is $7,316, which gives $365.80."
-us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model treated Louisiana's EITC as nonrefundable and capped at zero tax liability. The LA EITC is refundable at 5% of the federal credit and pays $365.80 even with no state tax owed.
-us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said the LA EITC is nonrefundable, so zero state liability left no refundable credit. The LA EITC is refundable at 5% of the $7,316 federal EITC, which is $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no Louisiana refundable credits and missed the refundable LA EITC. That credit applies automatically to any federal EITC recipient: 5% of $7,316 = $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed no Louisiana refundable credits at this income level. This low-income family with two children receives the maximum federal EITC of $7,316, which triggers Louisiana's refundable 5% EITC of $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for listed credit inputs and found none. It missed that the refundable LA EITC comes directly from the federal EITC the household earns, at 5% of $7,316 = $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the 5% Louisiana rate to a federal EITC of $8,231, which is the 2026 maximum for three or more children. The household has two children, so the federal EITC is $7,316 and the LA credit is $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the correct $7,316 federal EITC but applied an outdated 3.5% Louisiana rate instead of the current 5%. That gave $256.06 instead of $365.80."
-us,scenario_038,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but put the federal EITC at $7,427 instead of the 2026 two-child maximum of $7,316. That overstated the LA credit as $371.35 instead of $365.80."
-us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said the listed facts trigger no Louisiana refundable credits. The refundable LA EITC follows automatically from the household's $7,316 federal EITC, at 5% = $365.80."
-us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said Louisiana has no state EITC. Louisiana pays a refundable EITC of 5% of the federal credit, which is $365.80 for this household."
-us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model said Louisiana has no refundable EITC. Louisiana's EITC is refundable at 5% of the federal EITC, which is $365.80 on the $7,316 federal credit."
-us,scenario_038,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of $7,332 instead of the 2026 two-child maximum of $7,316. It also rounded to whole dollars, answering $367 instead of $365.80."
-us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model found no applicable Louisiana refundable credit and missed the refundable LA EITC, which requires no qualifying expenses. That credit is 5% of the $7,316 federal EITC, or $365.80."
-us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of about $7,422 instead of the 2026 two-child maximum of $7,316. That produced $371 instead of $365.80."
-us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so no numeric answer was scored. The correct derivation is 5% of the $7,316 federal EITC, or $365.80."
-us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of $6,500. That understates the 2026 two-child maximum of $7,316, which this household receives on the plateau."
-us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tied Louisiana refundable credits to positive state taxable income or child-care activity. The LA EITC is refundable at 5% of the federal EITC regardless of state liability, which is $365.80 here."
-us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly placed the household on the federal EITC plateau and applied the 5% rate. However, it used $7,430 as the 2026 two-child maximum instead of $7,316, which gave $371.50 instead of $365.80."
-us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said Louisiana offers no applicable refundable credit, omitting the refundable LA EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Used qualifying-surviving-spouse status (joint $32k/$44k SS thresholds, $32,200 standard deduction, joint brackets), but QSS under IRC §2(a) requires a dependent child and none is listed, so the head files single. It also put the $25,950 estate income into AGI and provisional income, whereas the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and adds a $3,319.81 QBI deduction, taxing $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Applied the joint $32,200 standard deduction and joint brackets under surviving-spouse status even though, with no dependent child, the head files single ($16,100 deduction, 10% bracket to $12,400). It also included the $25,950 estate income, which made 85% of SS taxable; the reference keeps estate income out of AGI (taxable SS $10,129.10) and takes a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that an elderly/disabled credit and a disability add-on to the standard deduction wipe out the tax. Disability does not increase the standard deduction, and the §22 credit is fully phased out at this income. It also mis-added AGI ($84,395); the reference taxes $13,279.25 at single rates for $1,345.51."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Filed single but used a stale $15,750 standard deduction and made-up brackets (10% to $12,150, 12% to $49,350) instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, so 85% of SS became taxable. The reference excludes estate income (taxable SS $10,129.10, AGI $32,699.07) and subtracts $16,100 plus a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Treated the head as a qualifying surviving spouse, using joint $32k/$44k SS thresholds, a ~$30,000 standard deduction and joint brackets, though no dependent child is listed and the head files single. It also included estate income in provisional income and AGI; the reference excludes it, giving taxable SS $10,129.10, AGI $32,699.07, a $3,319.81 QBI deduction and taxable income $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Used single status but a ~$16,700 standard deduction instead of $16,100, and its $6,210 does not follow from its own ~$62,500 of taxable income (2026 single rates give about $8,460). It also included the $25,950 estate income that the reference leaves out of AGI (taxable SS $10,129.10, AGI $32,699.07, QBI deduction $3,319.81)."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used the joint $32,200 standard deduction and joint brackets for surviving-spouse status even though no dependent child exists, so single status and the $16,100 deduction apply. It also counted the $25,950 estate income, which pushed SS to $30,689.25 taxable; the reference excludes it (taxable SS $10,129.10) and adds a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed qualifying-surviving-spouse status (joint $44k SS threshold, ~$30,000 deduction, 2025 joint $23,850 bracket), but that status requires a dependent child, so the head files single with the $16,100 deduction and 2026 single brackets. It also included estate income in provisional income and AGI, which the reference excludes (AGI $32,699.07, QBI deduction $3,319.81)."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse status with a joint ~$31,500 deduction and 2025 joint brackets, though no dependent child is listed and single status applies. It then fudged the result to $5,187, and it included the $25,950 estate income that the reference keeps out of AGI (taxable SS $10,129.10, taxable income $13,279.25)."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied the joint $32,200 standard deduction and joint brackets under surviving-spouse status even though the head has no dependent child and files single. It also counted estate income, which made 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and deducts $16,100 plus $3,319.81 of QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used the pre-OBBBA 2025 single standard deduction of $15,000 and 2025 brackets ($11,925/$48,475) instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, which the reference excludes from AGI (taxable SS $10,129.10) while applying a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA had sunset, using an $8,800 standard deduction and a 10%/15% schedule (and never reaching the 25% bracket). OBBBA made the TCJA rates and the $16,100 single deduction permanent for 2026. It also included the estate income that the reference leaves out of AGI (AGI $32,699.07)."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Used a $15,400 single standard deduction and $12,250/$49,800 brackets instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, making 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and subtracts a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Had the correct $16,100 deduction but wrong 2026 single bracket edges ($12,200/$49,600 instead of $12,400/$50,400). It also put the $25,950 estate income into AGI and provisional income; the reference excludes it, yielding taxable SS $10,129.10, a $3,319.81 QBI deduction and taxable income $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration law (a $13,750 standard deduction plus exemption and 10%/15%/25% brackets), but OBBBA made TCJA rates permanent, so 2026 uses the $16,100 deduction and 10%/12% brackets. It also included estate income that the reference excludes from AGI ($32,699.07)."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. Its $5,354 corresponds to taxable income of roughly $47,000–$49,000 at joint rates, which matches full inclusion of the estate income (85% of SS taxable) with a joint surviving-spouse deduction. The reference uses single status, excludes the estate income (AGI $32,699.07) and taxes $13,279.25 after the $16,100 deduction and a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"On its $79,209 AGI, $11,415 exceeds what 2026 single rates produce with the $16,100 deduction ($8,596). That places it on TCJA-sunset parameters (smaller deduction/exemption, 15%/25% brackets) rather than OBBBA's permanent schedule. It also included the estate income that the reference excludes from AGI ($32,699.07)."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly used post-TCJA-sunset parameters (a ~$13,400 standard deduction plus personal exemption), but OBBBA keeps the $16,100 single deduction and 10%/12%/22% brackets in 2026. It also included the $25,950 estate income that the reference leaves out of AGI and provisional income."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. $6,233 corresponds to about $52,400 of taxable income at 2026 single rates, meaning it counted the $25,950 estate income and 85% of SS. The reference excludes the estate income (taxable SS $10,129.10, AGI $32,699.07) and taxes only $13,279.25 after $16,100 standard and $3,319.81 QBI deductions."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used a $15,700 deduction instead of $16,100 and miscomputed tax on its own $63,509.25 of taxable income (2026 single rates give $8,684, not $7,117). It also included the estate income that the reference excludes from AGI (AGI $32,699.07, taxable income $13,279.25)."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"From AGI of $79,209, a $9,414 result implies a deduction of only about $12,400 instead of the $16,100 single standard deduction. It also counted the $25,950 estate income, which the reference excludes (taxable SS $10,129.10) while adding a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted estate income toward AGI, so SS hit the 85% cap. Its $7,578 corresponds to about $58,500 of taxable income, which does not match any correct 2026 single deduction. The reference excludes the estate income (AGI $32,699.07) and taxes $13,279.25 after the $16,100 and $3,319.81 QBI deductions."
-us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Estimated a $15,432 standard deduction and $12,259/$49,838 brackets instead of the published 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income that the reference excludes, which pushed taxable SS to $30,689.25 instead of $10,129.10."
-us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Dropped the estate income but also ignored the $6,260 SE loss and taxed a flat 85% of SS without the provisional-income test, which on non-estate income gives $10,129.10. It then invented a $1,920 'tax on previously-taxed Social Security' credit and skipped the $3,319.81 QBI deduction, instead of taxing $13,279.25 at 10%/12%."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Asserted that taxable income is not positive after the standard deduction and elderly/surviving-spouse treatment, but no age-65 or surviving-spouse relief applies at age 61 without a dependent. Even the reference's AGI of $32,699.07 leaves $13,279.25 taxable after the $16,100 and $3,319.81 QBI deductions, which is taxed at $1,345.51."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that deductions and nonrefundable credits eliminate the liability, but no credits apply and AGI of $32,699.07 exceeds the $16,100 deduction plus the $3,319.81 QBI deduction. That leaves $13,279.25 taxable at 10%/12%, or $1,345.51."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse joint brackets and the $32,200 deduction, but with no dependent child the head files single ($16,100 deduction). It also included the estate income, making 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and adds a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Applied surviving-spouse/MFJ treatment (joint $32k/$44k SS thresholds, $32,200 deduction, joint brackets) without a dependent child, so single status applies. It also included the $25,950 estate income in provisional income and AGI; the reference excludes it and taxes $13,279.25 after a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used qualifying-surviving-spouse joint SS thresholds, the $32,200 deduction and joint brackets, though that status requires a dependent child and the head files single. It also counted estate income that the reference excludes from AGI (taxable SS $10,129.10, AGI $32,699.07, taxable income $13,279.25)."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse joint status and then added an age-based additional standard deduction and the OBBBA senior deduction, both of which require age 65, for a 61-year-old. It also included the estate income; the reference uses single status, excludes estate income (AGI $32,699.07) and deducts $16,100 plus $3,319.81 QBI."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Applied surviving-spouse joint brackets and the $32,200 deduction even though no dependent child is listed, so single status and the $16,100 deduction apply. It also counted the $25,950 estate income, which drove taxable SS to $30,689.25 instead of the reference's $10,129.10."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Correctly filed single with the 2026 $16,100 deduction and brackets, but included the $25,950 estate income in AGI and provisional income (85% of SS taxable, AGI $79,209), producing $8,596. The reference leaves estate income out of AGI (taxable SS $10,129.10, AGI $32,699.07) and grants a $3,319.81 QBI deduction, so it taxes $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used single status, the $16,100 deduction and 2026 single brackets, and included the $25,950 estate income so SS was taxable at the $30,689.25 cap, giving $8,596.04. The reference excludes estate income from AGI and provisional income (taxable SS $10,129.10, AGI $32,699.07) and subtracts a $3,319.81 QBI deduction, leaving $13,279.25 taxable."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse status with the $32,200 joint deduction and joint brackets, though with no dependent child the head files single. It also included the $25,950 estate income that the reference excludes from AGI, where taxable SS is $10,129.10 and the QBI deduction is $3,319.81."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Gave a rounded $8,000 guess based on estate income, 85%-taxable SS and a medical-expense deduction. Medical expenses above the 7.5% floor fall far short of the $16,100 standard deduction, and the reference excludes estate income from AGI ($32,699.07) and applies a $3,319.81 QBI deduction, taxing $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Used TCJA-sunset parameters ($8,255 deduction plus $5,265 exemption, 10%/15%/25% brackets), but OBBBA made the $16,100 single deduction and 10%/12%/22% brackets permanent for 2026. It also included the $25,950 estate income, which the reference excludes from AGI ($32,699.07)."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA had sunset ($8,300 deduction plus $5,300 exemption, 15%/25% brackets) instead of applying OBBBA's permanent $16,100 single deduction and 10%/12% brackets. It also counted the estate income that the reference excludes (taxable SS $10,129.10, taxable income $13,279.25)."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset 2026 law ($8,500 deduction, $5,400 exemption, 15%/25% brackets), but OBBBA keeps the $16,100 single deduction and TCJA rate schedule. It also included the $25,950 estate income that the reference leaves out of AGI and provisional income."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Projected a $15,420 deduction and $12,259/$49,832 brackets instead of the published 2026 values of $16,100 and $12,400/$50,400. It also included estate income, so SS was 85% taxable; the reference excludes it (AGI $32,699.07) and adds a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Had the correct $16,100 deduction but wrong bracket amounts (about $1,225 in the 10% band and $4,509 in the 12% band instead of $1,240 and $4,560). It also included the $25,950 estate income, which the reference excludes from AGI (taxable SS $10,129.10) while adding a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $1,345.51 reference."
-us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Correctly rejected QSS status and applied single status, the $16,100 deduction and 2026 brackets, but included the $25,950 estate income in AGI and provisional income, reaching $8,596.04. The reference omits estate income from AGI (taxable SS $10,129.10, AGI $32,699.07) and grants a $3,319.81 QBI deduction, taxing $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Double-counted Social Security by adding the full $36,105 benefit and then 85% of it again. It also invented extra standard deductions for being 61 and for disability; neither exists, and the $16,100 single deduction is the only one. The reference's AGI is $32,699.07, which excludes estate income and includes only $10,129.10 of SS."
-us,scenario_039,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Correctly filed single with the $16,100 deduction and 2026 brackets but included the $25,950 estate income, which made SS taxable at $30,689 and AGI $79,209, for $8,596. The reference excludes estate income from AGI and provisional income (taxable SS $10,129.10) and subtracts a $3,319.81 QBI deduction, leaving $13,279.25 taxable."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Settled on qualifying-surviving-spouse treatment (joint $32k/$44k SS thresholds, $32,200 deduction, 2025 joint $23,850 bracket) without a dependent child, where single status applies. It also mis-multiplied the SS worksheet and included the estate income that the reference excludes from AGI ($32,699.07)."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Produced taxable income of $90,564, more than its own AGI, after subtracting a $31,900 deduction. It then invented a credit for other dependents funded by the pension and an AMT floor, none of which applies to a single filer with no dependents. The reference taxes $13,279.25 for $1,345.51 with no nonrefundable credits."
+us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly applied 5% to a federal EITC on the plateau. However, it used ~$7,478 as the 2026 two-child maximum EITC instead of $7,316, which overstated the Louisiana credit as $374 instead of $365.80."
+us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed Louisiana has no refundable EITC and only nonrefundable credits. In fact, Louisiana provides a refundable EITC equal to 5% of the federal EITC. With a $7,316 federal EITC, that yields $365.80."
+us,scenario_038,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model asserted that no refundable Louisiana credits apply. It missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used the correct 5% rate but put the federal EITC at about $3,680 for MFJ with two children and ~$21,400 of earned income. That income is on the plateau, below the ~$31k MFJ phase-out threshold, so the federal credit is the full $7,316. The Louisiana credit is therefore $365.80."
+us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model put the federal EITC at ~$2,640 as if ~$21,900 of earned income were deep in the phase-out range. For a joint filer that income sits on the plateau below the ~$31k MFJ phase-out start, so the federal EITC is $7,316 and 5% of it is $365.80."
+us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but rounded the federal EITC down to about $7,000. The 2026 two-child maximum, which this plateau-income household receives, is $7,316, so the credit is $365.80, not $350."
+us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana eliminated its refundable EITC. Louisiana's refundable EITC remains at 5% of the federal EITC, which gives $365.80 on the $7,316 federal credit."
+us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable and returned $0. The Louisiana EITC is refundable at 5% of the federal EITC, paying $365.80 regardless of state tax liability."
+us,scenario_038,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model correctly identified the $7,316 federal EITC but applied a 50% Louisiana match instead of the actual 5%. That inflated the credit tenfold, to $3,658 instead of $365.80."
+us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model made two errors: it used a 3.5% Louisiana rate instead of 5%, and a $7,430 federal EITC instead of the 2026 two-child maximum of $7,316. Correctly computed, 5% × $7,316 = $365.80."
+us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,354 as the federal EITC instead of the 2026 two-child maximum of $7,316. That overstated the credit by $1.90."
+us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no refundable Louisiana credit applies. It missed the refundable Louisiana EITC, which is 5% of the federal EITC: $365.80 on a $7,316 federal credit."
+us,scenario_038,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,The model claimed Louisiana eliminated its state EITC. Louisiana's refundable EITC at 5% of the federal credit is in effect for 2026 and yields $365.80 here.
+us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model returned $0 without recognizing the Louisiana refundable EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,250 as the federal EITC instead of the 2026 two-child maximum of $7,316. That understated the credit as $362.50 instead of $365.80."
+us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned $0 and ignored Louisiana's refundable EITC. That credit equals 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly identified the Louisiana EITC as a refundable share of the federal credit. However, $353.43 at 5% implies a federal EITC of about $7,069 instead of the $7,316 two-child plateau maximum for 2026."
+us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model recognized the refundable Louisiana EITC, but its $350 implies a $7,000 federal EITC. The household receives the 2026 two-child maximum of $7,316, so 5% gives $365.80."
+us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable and zeroed it out because state liability is $0. The LA EITC is refundable at 5% of the federal EITC, which pays $365.80 even with no state tax."
+us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model classified the Louisiana EITC as nonrefundable. The EITC is refundable regardless of state tax liability or child care expenses, and it yields 5% × $7,316 = $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model looked only for listed credit inputs and missed that Louisiana's refundable EITC is computed automatically from the federal EITC. At 5% × $7,316, the credit is $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed no Louisiana refundable credit at this income level. Low-to-moderate earned income is exactly what produces a federal EITC of $7,316, and Louisiana's refundable 5% match is $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model required explicit refundable-credit inputs and returned $0. Louisiana's refundable EITC is derived from the federal EITC the household qualifies for: 5% × $7,316 = $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $8,231, which is the 2026 maximum for three or more children. This household has two qualifying children, so the maximum is $7,316 and the credit is $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the correct $7,316 federal EITC but applied a 3.5% Louisiana rate instead of 5%. That gave $256.06 instead of $365.80."
+us,scenario_038,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but put the federal EITC at $7,427 instead of the 2026 two-child maximum of $7,316. That overstated the credit by $5.55."
+us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits are triggered. Louisiana's refundable EITC is triggered automatically by the federal EITC and pays 5% × $7,316 = $365.80."
+us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model asserted that Louisiana has no state EITC. Louisiana has a refundable EITC at 5% of the federal credit, which is $365.80 for this household."
+us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana has no refundable EITC. Louisiana's EITC is refundable at 5% of the $7,316 federal EITC, giving $365.80."
+us,scenario_038,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,332 as the federal EITC instead of the 2026 two-child maximum of $7,316. It also rounded the result to whole dollars, giving $367 instead of $365.80."
+us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model concluded there was no applicable EITC match. Louisiana's refundable EITC matches 5% of the federal EITC and yields $365.80 on this household's $7,316 federal credit."
+us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used ~$7,422 as the federal EITC instead of the 2026 two-child maximum of $7,316. That overstated the credit as $371 instead of $365.80."
+us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there is no answer to score. The correct derivation is 5% of the $7,316 federal EITC, which is $365.80."
+us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of $6,500. The household's federal EITC is the 2026 two-child plateau maximum of $7,316, so the credit is $365.80, not $325."
+us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tied refundable credits to Louisiana taxable income and school-readiness activities. It missed that the Louisiana EITC is refundable regardless of state taxable income and pays 5% × $7,316 = $365.80."
+us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly placed the household on the federal EITC plateau but used $7,430 as the 2026 two-child maximum instead of $7,316. That gave $371.50 instead of $365.80."
+us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model stated that Louisiana provides no applicable refundable credits. It missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model actually reached the correct single-filer result of $8,596, then dropped it. It switched to qualifying-surviving-spouse treatment even though there is no dependent child, used the MFJ $32,000/$44,000 Social Security thresholds (taxable SS $25,187) and the $32,200 joint deduction, and ended at about $4,488."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It applied qualifying-surviving-spouse joint brackets and the $32,200 joint standard deduction, but that status requires a dependent child and none is listed. The correct treatment is single: $16,100 deduction, taxable income $63,109, with the top slice taxed at 22% above $50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It overstated AGI at $84,395 instead of $79,209 and invented an extra standard deduction for disability. It then claimed an elderly/disabled credit wipes out the tax, but that credit is fully phased out at this AGI and Social Security level. Correct single taxable income is $63,109, which owes $8,596."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and add-ons leave no taxable income. In fact AGI is $79,209 and only a $16,100 single deduction applies, because there is no age-65 add-on at 61 and no disability add-on. That leaves $63,109 taxable and $8,596 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Filing status was correct (single), but it added up AGI as $75,209 instead of $79,209, dropping $4,000. It also used the pre-OBBBA $15,750 deduction and estimated brackets ($12,150/$49,350) instead of the 2026 values of $16,100 and $12,400/$50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying widow(er) with no dependent child. That led it to the MFJ $32,000/$44,000 Social Security thresholds (taxable SS $25,187 instead of $30,689), a roughly $30,000 joint deduction, and joint brackets. Single filing with a $16,100 deduction is correct."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"AGI of $79,209 and single treatment were right, but it used a $16,700 deduction instead of $16,100. It also put about $62,500 of taxable income through the brackets and got $6,210, when the single schedule (10% to $12,400, 12% to $50,400, 22% above) gives about $8,460 on that amount."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It filed as surviving spouse with joint brackets and the $32,200 joint deduction, but that status requires a dependent child and the household has none. Single treatment with a $16,100 deduction and the 22% bracket above $50,400 gives $8,596."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It assumed qualifying-surviving-spouse status, which requires a dependent child and none exists. That led it to the MFJ Social Security thresholds (taxable SS $25,187), an estimated $30,000 joint deduction, and joint brackets with a $23,850 top for the 10% band. Single filing with a $16,100 deduction is correct."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used qualifying-surviving-spouse joint brackets and a roughly $31,500 joint deduction even though there is no dependent child. It also fudged its own $5,248 result down to $5,187. Correct single treatment gives taxable income of $63,109 and $8,596 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It applied the $32,200 surviving-spouse joint deduction and joint brackets without the dependent child that status requires. Filing single gives a $16,100 deduction, $63,109 of taxable income, and $8,596 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Single status and the $79,209 AGI were right, but it used the pre-OBBBA $15,000 standard deduction and 2025 brackets ($11,925/$48,475). The 2026 values are a $16,100 deduction and brackets at $12,400/$50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed TCJA expired and applied an $8,800 deduction with pre-TCJA 10%/15% brackets. OBBBA made the TCJA structure permanent, so 2026 uses a $16,100 single deduction and 10%/12%/22% rates."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Single status and AGI were right, but it used a $15,400 deduction and brackets at $12,250/$49,800. The 2026 values are $16,100 and $12,400/$50,400, so it overstated taxable income by $700 and taxed too much at 22%."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"AGI, the $16,100 deduction, and $63,109 of taxable income were all correct. It used bracket thresholds of $12,200 and $49,600 instead of the 2026 single thresholds of $12,400 and $50,400, which pushed $800 too much into the 22% bracket."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA sunset in 2026 and applied a combined $13,750 deduction and personal exemption with 15%/25% brackets. OBBBA kept the TCJA rates and set the 2026 single deduction at $16,100, so the 10%/12%/22% schedule applies."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It showed no computation. Its $5,354 sits about $3,240 below the correct single-filer $8,596, which matches joint-rate surviving-spouse treatment: a roughly $30,000–$32,200 deduction with brackets capping 10% at $24,800. Without a dependent child the head files single."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"With single status and $79,209 of AGI, its $11,415 matches TCJA-sunset parameters: a small deduction plus personal exemption and 15%/25% rates. OBBBA made TCJA permanent, so the correct figures are the $16,100 deduction and 10%/12%/22% brackets, giving $8,596."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied 'post-TCJA' parameters, a roughly $13,400 deduction plus exemption with pre-TCJA rates, on the assumption that TCJA expired. OBBBA kept the TCJA structure, so the correct inputs are the $16,100 single deduction and 10%/12%/22% brackets."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $6,233 nearly matches head-of-household treatment (a $24,150 deduction with 12% running to $67,450, which yields about $6,250). A surviving spouse with no dependent files single, with a $16,100 deduction and $8,596 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"AGI was right, but it used a $15,700 deduction instead of $16,100. On its $63,509 of taxable income it reported $7,117, which matches joint-rate brackets (10% to $24,800, then 12%). The single schedule taxes everything above $50,400 at 22%."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"With the correct $79,209 of AGI, its $9,414 overshoots by $818. That equals 22% on about $3,700 of extra taxable income, which matches using a deduction near $12,400 instead of the 2026 single deduction of $16,100."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It gave no numbers. Its $7,578 is $1,018 below the correct figure, which works out to roughly 12% of $63,109 of taxable income. That suggests it never applied the 22% rate to the $12,709 above the $50,400 single threshold."
+us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Single status and AGI were right, but it estimated a $15,432 deduction and brackets at $12,259/$49,838 instead of the enacted 2026 values: $16,100 and $12,400/$50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It left the $25,950 of estate income and the $6,260 self-employment loss out of gross income. It also justified the deduction by calling the 61-year-old 65 or older, and subtracted an invented $1,920 'tax on previously-taxed Social Security' credit."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction plus elderly/surviving-spouse treatment leaves no taxable income. There is no elderly add-on at 61, and the single $16,100 deduction against $79,209 of AGI leaves $63,109 taxable, which owes $8,596."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It asserted that nonrefundable credits wipe out the liability. No nonrefundable credit applies here: there are no dependents, and the elderly/disabled credit is fully phased out at this AGI. Tax on $63,109 of single taxable income is $8,596."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint brackets and the $32,200 deduction, but that status requires a dependent child and none is listed. Single filing gives a $16,100 deduction and $8,596 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse/MFJ treatment without a dependent child. That brought in the MFJ Social Security thresholds (taxable SS $25,187 instead of $30,689), the joint deduction, and joint brackets, instead of single filing."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse even though there is no dependent child. It used the joint Social Security thresholds (taxable SS $25,187) and the $32,200 deduction, when the single thresholds, the $16,100 deduction, and single brackets apply."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It used the joint surviving-spouse deduction without a dependent child. It also added the age-65 additional deduction and the OBBBA senior deduction for a 61-year-old, and both require being 65 or older. The correct deduction is $16,100 for a single filer."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It filed as surviving spouse with the $32,200 joint deduction and joint brackets, but the household has no dependent child, so single filing with a $16,100 deduction is correct. That makes taxable income $63,109, not $47,009."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse status, which requires a dependent child, and used the $32,200 deduction and joint brackets. Single filing gives $63,109 of taxable income, with the top $12,709 taxed at 22%."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted medical expenses on top of the standard deduction, which is not allowed: itemized medical after the 7.5% floor is only about $3,794, so the $16,100 standard deduction is taken instead. It also reported a rounded $8,000 rather than running $63,109 through the 2026 single brackets."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset parameters: an $8,255 deduction plus a $5,265 personal exemption, with 15%/25% rates. OBBBA made TCJA permanent, so the 2026 single deduction is $16,100 with no exemption, and the rates are 10%/12%/22%."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset and used an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets. Under OBBBA, 2026 keeps the $16,100 single deduction and 10%/12%/22% brackets at $12,400/$50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA-sunset, with an $8,500 deduction, a $5,400 exemption, and 15%/25% rates. OBBBA made the TCJA rate structure permanent, so the correct inputs are the $16,100 single deduction and 10%/12%/22% brackets."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Single status and AGI were right, but it used a projected $15,420 deduction and brackets at $12,259/$49,832 instead of the enacted 2026 values: $16,100 and $12,400/$50,400."
+us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The $16,100 deduction and $63,109 of taxable income were correct. It used bracket tops of about $12,250 and $49,825 instead of $12,400 and $50,400, so it taxed about $575 too much at 22% and got $8,656."
+us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no answer and no explanation for this variable, so there was no value to score."
+us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It double-counted Social Security by adding the full $36,105 benefit and then the $30,689 taxable portion again, inflating taxable income to about $99,214. It also invented extra standard deductions for being 61 and disabled, which do not exist."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It settled on qualifying-surviving-spouse/MFJ treatment without a dependent child. That brought in the MFJ Social Security thresholds (taxable SS about $25,197), the $32,200 deduction, and joint brackets, instead of single filing with a $16,100 deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported taxable income of $90,564, which exceeds AGI even after its $31,900 joint-style deduction. It halved the self-employment loss and invented a $1,716 credit for other dependents tied to pension income, even though there are no dependents. Correct single taxable income is $63,109."
us,scenario_039,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"Virginia's expansion adult group covers 19-64-year-olds only up to 138% FPL, yet the model declared that countable MAGI ""falls within eligible range for a household size of 1"" without ever summing the listed income: $36,105 of Social Security counts in full under MAGI rules, on top of $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, against which the $6,260 self-employment loss is trivial, putting the head at 368% FPL. Its correct observation that the expansion group applies no asset test is irrelevant once income blows through the 138% FPL ceiling, and the SSI-related aged/blind/disabled group it gestured at requires income near the SSI standard with SSI receipt at $0 here."
us,scenario_039,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model named the correct rule — Virginia expanded to 138% FPL — then asserted the head's MAGI was below that threshold without computing it. Medicaid MAGI here is 368% FPL: full Social Security of $36,105, $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, less the $6,260 self-employment loss. With income at 2.7x the expansion limit and no SSI receipt to open the aged/blind/disabled route, the head qualifies through no Virginia category."
us,scenario_039,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the ""is disabled"" flag as a standalone categorical qualifier and explicitly declined to weigh the income facts, stating no disqualifying income was provided when the prompt lists $36,105 Social Security, $26,800 IRA distributions, $2,030 pension, and $25,950 estate income. Every disability-linked Virginia pathway is income-tested — SSI-linked eligibility requires SSI receipt, which is $0, and the ABD and expansion groups cap income at or below 138% FPL against this head's 368% FPL — so disability determines which income test applies, not whether eligibility is automatic."
-us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"Treated the $36,105 of Social Security retirement income as if it were disability benefits. Medicare's under-65 pathway requires 24 months of SSDI entitlement. The head has no SSDI and no SSDI months (both count as 0), so the disability pathway fails. The head is under 65, so the age pathway fails too."
-us,scenario_039,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"Claimed the head qualifies through the disability pathway because they are disabled and receive Social Security. That income is retirement income, not SSDI. The pathway requires at least 24 months of SSDI receipt, and the head has 0 months, so they are not eligible at 61."
-us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"Said disabled people are eligible for Medicare regardless of age, and read Social Security retirement income as proof of benefit eligibility. The under-65 pathway requires 24 months of Social Security disability receipt, or ESRD/ALS. The head has none of these, so a 61-year-old is not eligible."
-us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"Named the correct rule (Medicare after 24 months of SSDI) and then did not apply it. The head has no SSDI income and no SSDI months, so the 24-month requirement is not met, and the head is not eligible."
-us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"Treated disabled status alone as enough for Medicare before 65. The disability pathway requires 24 months of SSDI entitlement, and the head has none. At 61 with no ESRD or ALS, the head is not eligible."
-us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Assumed PolicyEngine gives Medicare eligibility to anyone flagged disabled. The under-65 rule instead requires at least 24 months of Social Security disability receipt, or ESRD/ALS. The head has 0 SSDI months and neither condition, so they are ineligible."
-us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Stated the 24-month SSDI requirement and then skipped it, going straight from 'is disabled' to eligible. The head receives retirement benefits, not SSDI, and has 0 SSDI months. Surviving-spouse status is not a Medicare pathway for someone under 65, so the head is not eligible."
-us,scenario_039,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Assumed PolicyEngine treats disabled people as Medicare eligible. Eligibility under 65 actually requires 24 or more months of SSDI receipt, or ESRD/ALS. The head has 0 SSDI months, so a 61-year-old is not eligible."
-us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"Treated 'age 61 and disabled' as meeting the Medicare requirements. Nothing in the facts satisfies the under-65 pathway, which requires 24 months of SSDI entitlement (or ESRD/ALS). The head has no SSDI, so they are not eligible."
-us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"Took the disabled flag as a qualifying condition. Medicare does not look at disability status by itself; under 65 it requires 24 months of Social Security disability benefits or ESRD/ALS. The head has none of these, so they are ineligible."
-us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"Assumed coverage starts once someone is disabled. Medicare disability coverage starts only after 24 months of SSDI entitlement. The head has 0 months of SSDI receipt, so the head is not eligible at 61."
-us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"Applied a disability rule that treats the flag alone as enough. The real rule requires at least 24 months of SSDI receipt, and the head has 0 because only retirement Social Security is listed. The head is therefore not eligible under 65."
-us,scenario_039,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Said the stated disability makes the head eligible, but the modeled rule depends on months of SSDI receipt (24 or more), not on disability status. With 0 SSDI months and age 61, the head is not eligible."
-us,scenario_039,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"Concluded that disability gives Medicare before 65. That pathway requires 24 months of Social Security disability entitlement, and the head has no SSDI income or months. At 61 the head is not eligible."
-us,scenario_039,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"Relied on a disability-based Medicare rule, but that rule is triggered by 24 or more months of SSDI receipt, not by the disabled flag. The head's SSDI months are 0, so they are not eligible at 61."
-us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"Went straight from 'disabled' to Medicare eligibility without checking the 24-month SSDI requirement. The head has no SSDI receipt and no ESRD/ALS, so at 61 they are not eligible."
-us,scenario_039,head_medicare_eligible,grok-4.7,llm_error,age_disability,False,"Assumed PolicyEngine counts disability itself as a Medicare qualifier. PolicyEngine's under-65 test uses months of Social Security disability receipt (24 or more) or ESRD/ALS, all of which are 0 or false here. The head is not eligible."
-us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"Stated Medicare eligibility as 'age 65+ or disabled'. The disability prong actually requires 24 months of SSDI entitlement, and the head has 0 months. At 61 the head is ineligible."
-us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"Named the 24-month SSDI rule and then treated the disabled flag as meeting it. No SSDI income or SSDI months are listed, so both are 0, and the head's Social Security is retirement income. The head is not eligible at 61."
+us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated 'disabled plus receiving Social Security' as qualifying for Medicare. The head's $36,105 is Social Security retirement income, not SSDI, and the head has zero months of SSDI receipt against the 24 required. At age 61 with no ESRD, the head is not eligible."
+us,scenario_039,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model said disability plus Social Security receipt meets PolicyEngine's disability pathway. That pathway requires 24 months of Social Security Disability Insurance, and the head only has retirement benefits with zero SSDI months. A 61-year-old without ESRD is not eligible."
+us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model claimed disabled people are Medicare-eligible regardless of age and took Social Security retirement income as proof of benefit eligibility. Under-65 eligibility requires 24 months of SSDI receipt specifically, and this head has none. The retirement benefit gives no Medicare entitlement before 65."
+us,scenario_039,head_medicare_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model correctly stated that the disability pathway requires 24 months of SSDI, then ignored that requirement and answered yes based on the disabled flag. The head has zero SSDI months (unlisted inputs are 0), so the requirement is not met and the head is ineligible at 61."
+us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model noted the 24-month SSDI requirement but never checked it against the household facts. The head receives no SSDI (their Social Security is retirement income), so the months-of-SSDI test fails and a 61-year-old without ESRD is not Medicare-eligible."
+us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model concluded 'given disabled status, eligible' without applying the qualifying condition for under-65 disability Medicare: at least 24 months of SSDI receipt. The head has zero SSDI months and no ESRD, so at 61 they are not eligible."
+us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model assumed PolicyEngine treats the disabled flag as granting Medicare before 65. PolicyEngine's rule actually requires age ≥ 65, at least 24 months of SSDI, or ESRD. The head meets none of these, since their Social Security is retirement income rather than SSDI."
+us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model stated the 24-month SSDI requirement, then granted eligibility based on the disabled label alone and also cited surviving-spouse status. Neither creates Medicare entitlement at 61. The head has zero SSDI months, only retirement-type Social Security, and no ESRD."
+us,scenario_039,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model asserted that PolicyEngine treats disabled people as Medicare-eligible. PolicyEngine's under-65 path requires at least 24 months of SSDI receipt (or ESRD). The head has no SSDI months, so at 61 they are not eligible."
+us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model treated being 61 and disabled as meeting Medicare requirements. The under-65 disability pathway requires 24 months of SSDI receipt, which the head lacks because their Social Security is retirement income. With no ESRD, the head is ineligible."
+us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model treated 'qualifying disability' as met by the disabled flag. Qualification under 65 means at least 24 months of SSDI entitlement, and the head has zero SSDI months with only retirement Social Security. A 61-year-old without ESRD is not eligible."
+us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model assumed Medicare coverage starts as soon as a person is disabled. The rule requires 24 months of SSDI receipt before Medicare begins, and the head has no SSDI months. With no ESRD at age 61, the head is not eligible."
+us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model applied a disability rule for under-65 Medicare without its actual condition: at least 24 months of SSDI receipt. The head's Social Security is retirement income with zero SSDI months, so the head is ineligible at 61."
+us,scenario_039,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model said the stated disability qualifies under the modeled Medicare rules. The modeled rules test months receiving SSDI (at least 24), not the disabled flag. The head has zero SSDI months and no ESRD, so at 61 they are not eligible."
+us,scenario_039,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"The model treated disability by itself as granting Medicare before 65. Before 65, eligibility requires at least 24 months of SSDI receipt or ESRD, and the head has neither. Their $36,105 is Social Security retirement income."
+us,scenario_039,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model cited PolicyEngine's disability-based Medicare rule but misread its trigger. That rule keys on at least 24 months of SSDI receipt, not the disabled flag. The head has no SSDI months and no ESRD, so at 61 they are not eligible."
+us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model equated 'disabled' with Medicare-eligible. Under-65 eligibility requires 24 months of SSDI entitlement, and this head receives only Social Security retirement income with zero SSDI months, so they are ineligible at 61."
+us,scenario_039,head_medicare_eligible,grok-4.7,llm_error,age_disability,False,"The model assumed PolicyEngine treats disability as qualifying for Medicare. PolicyEngine requires age ≥ 65, at least 24 months of SSDI, or ESRD. The head is 61 with zero SSDI months and no ESRD, so they are not eligible."
+us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model stated the rule as 'age 65+ or disabled,' which drops the actual condition for the disability path: at least 24 months of SSDI receipt. The head has no SSDI months, so the disabled flag does not make them eligible at 61."
+us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model acknowledged that Medicare starts only after 24 months of SSDI, then answered yes anyway based on the disabled flag. The head has zero SSDI months and receives retirement Social Security, so the 24-month test fails and the head is ineligible."
us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage earnings and applied the 7.65% employee Social Security and Medicare rate to an unstated payroll-tax base. The household has no listed wage or salary income, while its listed estate, retirement, IRA, pension, and negative self-employment income are not subject to employee payroll tax, yielding $0."
us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract."
us,scenario_039,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Put the $25,950 of estate income into AGI, which gave a Virginia AGI of $48,520 instead of $22,569.97. It then left out the $930 personal exemption and, with no stated basis, raised its own bracket result of about $2,029 to $2,192. The reference taxes $12,889.97 after the $8,750 deduction and $930 exemption, for $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Gave the surviving spouse the $17,500 joint standard deduction, but Virginia has no surviving-spouse status, so the filer gets the single $8,750 deduction. It also counted the $25,950 of estate income in AGI (Virginia AGI $48,520 instead of $22,569.97), so its taxable income of $30,090 is far from the reference's $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Relied on a Virginia retirement-income credit and disability exclusion that do not exist in the form it described. A 61-year-old gets no age deduction (that starts at 65), and the disability subtraction covers only disability pay received in place of wages, not IRA or pension income. After the Social Security subtraction, $8,750 deduction and $930 exemption, $12,889.97 stays taxable, so tax is $514.50, not $0."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used a joint-level $17,000 standard deduction for the surviving spouse instead of Virginia's single $8,750, and counted the $25,950 of estate income in AGI. It also got the brackets wrong, putting the tax at $17,000 at $670 instead of $720."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI (Virginia AGI $48,520), used the outdated $8,500 deduction, and added an $800 exemption for disability, although Virginia gives the extra $800 only for age 65+ or blindness. It then cut its own $1,944 result to $1,545 for unnamed 'medical/other adjustments'."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI, which made taxable Social Security $30,689 and Virginia AGI $48,520. PolicyEngine's AGI of $32,699.07 has no estate income, so its taxable Social Security is $10,129.10 and Virginia taxable income is $12,889.97. The model also used the 2024 deduction of $8,500 instead of $8,750 and treated the exemption as a $30 credit."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in federal AGI, giving taxable Social Security of $30,689.25 and Virginia AGI of $48,520. PolicyEngine's AGI of $32,699.07 has no estate income, which leaves taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied a $20,000 retirement subtraction for a disabled filer to the pension and IRA income, but Virginia's disability subtraction covers only disability income paid in place of wages. It also started from gross Social Security instead of the taxable portion, used the joint $17,000 deduction, and claimed three $800 exemptions instead of one $930 exemption."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI and left out the $930 personal exemption. It then raised its own bracket result of $2,044 to $3,400 with no rule to support it. The reference taxes $12,889.97 for $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used the $17,500 joint standard deduction for a surviving spouse, but Virginia applies the single $8,750 deduction. It also counted the $25,950 of estate income in Virginia AGI ($48,520 instead of $22,569.97)."
-us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the full $36,105 of Social Security from an AGI that included only $30,689 of taxable benefits. It added a $12,000 retirement subtraction for disability that Virginia does not provide, and used an outdated $4,000 deduction and a $970 exemption instead of $8,750 and $930."
-us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Never applied Virginia's full subtraction of federally taxable Social Security and taxed a federal AGI of $79,209 that also included the $25,950 of estate income. The result was $69,529 of taxable income instead of $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI (taxable Social Security $30,689.25, Virginia AGI $48,520). PolicyEngine's AGI of $32,699.07 leaves it out, giving a Virginia AGI of $22,569.97. The model also used the 2024 deduction of $8,500 instead of the 2026 amount of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, while PolicyEngine's AGI is $32,699.07 without it. That difference raised taxable Social Security and Virginia AGI to $48,520. The model also used the outdated $8,500 deduction instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out, so its Virginia AGI was $48,520 instead of $22,569.97. It also used the 2024 deduction of $8,500 instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. $2,420 implies Virginia taxable income of about $46,565, which fits counting the estate income and taking almost none of the $8,750 deduction and $930 exemption. The reference taxes $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"$1,990 matches a Virginia AGI of $48,520 (estate income included) less the 2024 deduction of $8,500 and a $930 exemption. PolicyEngine's AGI leaves out the $25,950 of estate income, giving a Virginia AGI of $22,569.97 and taxable income of $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in federal AGI, which PolicyEngine's AGI leaves out, so its Virginia AGI was $48,520 instead of $22,569.97. It also used a $9,000 deduction and an $800 exemption instead of Virginia's $8,750 and $930."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. $3,371 implies Virginia taxable income of about $63,100, which fits taxing federal AGI including estate income without Virginia's full subtraction of taxable Social Security. The reference taxes $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the old $8,000 deduction instead of the 2026 amount of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"$1,990 matches a Virginia AGI of $48,520 that includes the $25,950 of estate income, less the outdated $8,500 deduction and a $930 exemption. PolicyEngine's AGI leaves out estate income, so Virginia taxable income is $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97. It also used the pre-2024 $8,000 deduction instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in non-Social Security income ($48,520), which made provisional income $66,572.50 and taxable Social Security $30,689.25. PolicyEngine leaves estate income out of AGI, which gives provisional income of $40,622.47, taxable Social Security of $10,129.10 and a Virginia AGI of $22,569.97. The model also used $8,500 instead of the $8,750 deduction."
-us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Treated Virginia as having a flat 2% rate instead of its 2%/3%/5%/5.75% brackets, and set the $6,260 self-employment loss to $0. It then added $2,000 of made-up 'prior-year estimated payments' to reach $2,516."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Assumed age-based relief would wipe out Virginia taxable income, but the age deduction starts at 65 and the filer is 61. After the Social Security subtraction, the $8,750 deduction and the $930 exemption, $12,889.97 stays taxable."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Assumed a low-income case with no tax and did no computation. Virginia AGI of $22,569.97 less $9,680 of deduction and exemption leaves $12,889.97 taxable, for $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used $48,520 of non-Social Security income that includes the $25,950 of estate income, which PolicyEngine's AGI leaves out, giving a Virginia AGI of $22,569.97. It also used the 2024 deduction of $8,500 instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Applied a surviving-spouse/joint standard deduction, but Virginia taxes a surviving spouse as single with the $8,750 deduction. It also counted the $25,950 of estate income in Virginia AGI."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used the $17,500 joint standard deduction for a surviving spouse instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI ($48,520 instead of $22,569.97)."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Applied a larger surviving-spouse standard deduction (about $12,900 implied) instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI. The reference taxes $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Used a $17,500 surviving-spouse standard deduction, but Virginia applies the single $8,750. It also counted the $25,950 of estate income, so its Virginia AGI was $48,520 instead of $22,569.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Taxed $69,529, which is federal AGI including estate income with no Virginia Social Security subtraction. Virginia fully subtracts federally taxable Social Security, and the reference's taxable income is $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI, giving taxable Social Security of $30,689.25 and $48,520 of remaining income. PolicyEngine's AGI of $32,699.07 leaves estate income out, which gives taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used a $17,500 surviving-spouse standard deduction instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI. It also made an arithmetic error: tax on $30,090 under its own brackets is $1,472.68, not $1,442.68."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed the standard deduction and exemptions cancel out all income. The $8,750 deduction plus the $930 exemption is only $9,680, which leaves $12,889.97 of the $22,569.97 Virginia AGI taxable, for $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97. It also used the old $8,000 deduction instead of the 2026 amount of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the $8,500 deduction instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97 and taxable income of $12,889.97. It also used the 2024 deduction of $8,500 instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out. It also used the pre-2024 $8,000 deduction instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of about $79,209 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the outdated $8,000 deduction instead of $8,750."
-us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $514.50 reference."
-us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, giving taxable Social Security of $30,689.25. PolicyEngine's AGI of $32,699.07 leaves estate income out, which gives taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference."
-us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Gave a 61-year-old age 65+ deductions and an extra 'surviving spouse age 65' adjustment, and never subtracted taxable Social Security. It then cut its own $4,100 estimate to $1,500 with no computation."
-us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Wrongly required AGI of $50,000 or less for Virginia's Social Security subtraction. Virginia subtracts all federally taxable Social Security at any income level, so the model taxed $69,779 instead of $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Put all $36,105 of Social Security into federal AGI instead of only the taxable portion. It used the $930 exemption as the only deduction, leaving out the $8,750 standard deduction, and then reported $3,260.45, which does not match its own computed $1,878.93."
-us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Made up an age-61 exemption of $11,898 and a nonrefundable 'pension deduction' credit of $2,030. Virginia has neither, and the reference's $514.50 comes only from the Social Security subtraction, the $8,750 deduction, the $930 exemption, and the brackets."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the $8,750 standard deduction but left out the $930 personal exemption, so its taxable income was $39,770 and its tax about $2,029, not $1,975.80. It then reported $2,192, which none of its own steps produce."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"The model gave the surviving spouse the $17,500 married-filing-jointly standard deduction. With no dependent, the head files as single and gets $8,750. That cut taxable income from $38,839.97 to $30,090."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model invented a Virginia retirement income credit and disability exclusion that wiped out the tax. Virginia has neither for a 61-year-old receiving IRA, pension and estate income. After the Social Security subtraction, the $8,750 standard deduction and the $930 exemption, $38,839.97 is still taxable."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,age_disability,False,"The model applied an age 65+ / disability deduction to zero out the tax. Virginia's age deduction requires age 65, and the disability subtraction covers only disability income, which this household doesn't have. Taxable income stays at $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as a joint filer and used a $17,000 standard deduction. With no dependent, the head files as single with an $8,750 deduction. Its bracket arithmetic was also inconsistent, giving $1,453 instead of tax on $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model used the outdated $8,500 standard deduction and added an $800 exemption for disability. Virginia's additional $800 exemption is only for age 65+ or blindness. It then cut its own $1,944 result to $1,545 for unspecified 'medical/other adjustments' that don't exist under the standard deduction."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the pre-2025 $8,500 single standard deduction instead of the 2026 amount of $8,750. That overstated taxable income by $250 and the tax by about $14."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model invented a $20,000 disability subtraction for pension and IRA income; Virginia's disability subtraction covers only disability income. It also used the $17,000 joint standard deduction for a single filer and claimed three $800 exemptions instead of one $930 exemption, which drove the tax down to $326."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model left out the $930 personal exemption and used $8,500 instead of the $8,750 standard deduction, which gives $2,044. It then raised that figure to $3,400 for an 'underestimated' Social Security amount, even though Virginia subtracts taxable Social Security in full."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"The model assumed a $17,500 joint standard deduction for the surviving spouse. With no dependent, the head files as single with $8,750, so taxable income is $38,839.97, not $30,090."
+us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $36,105 of Social Security, but federal AGI included only $30,689 of it. It also invented a $12,000 retirement subtraction for disability and used outdated $4,000 and $970 amounts instead of the $8,750 standard deduction and $930 exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model never applied Virginia's subtraction of the $30,689.25 of federally taxable Social Security. It taxed $69,529 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That produced $39,090 of taxable income instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied an $8,500 standard deduction instead of Virginia's 2026 single amount of $8,750. That overstated taxable income by $250 and the tax by about $14."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an $8,500 standard deduction instead of the 2026 single amount of $8,750. That gives $39,090 of taxable income instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no specific computation. The correct taxable income is $38,839.97, after subtracting taxable Social Security, the $8,750 standard deduction and the $930 exemption. Its $2,420 implies taxable income of about $46,565, so it didn't apply the full deductions."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $1,990 matches the outdated $8,500 single standard deduction. The 2026 amount is $8,750."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $9,000 standard deduction and an $800 personal exemption instead of the 2026 amounts of $8,750 and $930. That gave taxable income of $38,720 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no reasoning. Its $3,371 implies taxable income of about $63,100, far above the correct $38,839.97, so it didn't fully subtract the $30,689.25 of federally taxable Social Security from federal AGI."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction, which is outdated; the 2026 single amount is $8,750. That overstated taxable income by $750."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $1,990 matches the outdated $8,500 standard deduction. Virginia's 2026 single standard deduction is $8,750."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of the 2026 single amount of $8,750, so taxable income came out at $39,590 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model handled the Social Security subtraction correctly but used the outdated $8,500 standard deduction instead of $8,750. That gave $39,090 of taxable income."
+us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model left out the $25,950 of estate income and treated the self-employment loss as zero. It applied a flat 2% rate instead of Virginia's graduated brackets, which top out at 5.75%. It then added $2,000 of fictitious estimated payments to reach $2,516."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model relied on age-based treatment to make the tax base zero or negative. Virginia's age deduction starts at 65, so this 61-year-old gets none, and $38,839.97 remains taxable after the standard deduction and exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed a low-income scenario. Virginia AGI is $48,519.97 even after subtracting Social Security, so $38,839.97 is taxable after the $8,750 standard deduction and $930 exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That overstated taxable income by $250."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model gave the head a surviving-spouse/joint-based standard deduction. Without a dependent, the head files as single with $8,750. Its $1,760.18 implies taxable income of about $35,090 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 joint standard deduction for the surviving spouse. With no dependent, the head files as single with $8,750, so taxable income is $38,839.97, not $30,090."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model applied a 'surviving-spouse' standard deduction larger than the $8,750 single amount. A surviving spouse without a dependent files as single. Its $1,789 implies about $35,590 of taxable income instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 surviving-spouse/joint standard deduction. The head has no dependent and files as single with $8,750."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model taxed about $69,529 because it never subtracted the $30,689.25 of federally taxable Social Security from federal AGI. The correct taxable income is $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 surviving-spouse/joint standard deduction instead of the $8,750 single amount; the head has no dependent. It also miscalculated the bracket tax on its own $30,090, getting $1,442.68 instead of $1,472.68."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction and exemptions wipe out taxable income. Virginia AGI is $48,519.97 after the Social Security subtraction, and the $9,680 of deduction plus exemption leaves $38,839.97 taxable."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an $8,000 single standard deduction instead of the 2026 amount of $8,750, so taxable income was $39,590 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly ruled out the age deduction but used the outdated $8,500 standard deduction instead of $8,750, which put taxable income at $39,090."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used an $8,500 standard deduction instead of the 2026 single amount of $8,750. That overstated taxable income by $250 and the tax by about $14."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of the 2026 single amount of $8,750, which produced $39,590 of taxable income."
+us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of $8,750, so taxable income was $39,590 instead of $38,839.97."
+us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model started from an $84,625 AGI without subtracting Social Security. It invented age 65+, disability and surviving-spouse deductions that don't apply to a 61-year-old, then replaced its own ~$4,100 result with an arbitrary $1,500."
+us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model wrongly said Virginia's Social Security subtraction requires AGI of $50,000 or less. Virginia subtracts all federally taxable Social Security regardless of income, so $30,689.25 comes out of AGI. It also used the outdated $8,500 standard deduction instead of $8,750."
+us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $930 personal exemption as the standard deduction and left out the $8,750 standard deduction. It also built federal AGI with the full $36,105 of Social Security instead of the $30,689 taxable portion. It then reported $3,260.45, which doesn't match its own $1,878.93 computation."
+us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"The model invented an 'age-61 exemption' and a nonrefundable 'pension deduction' credit that cancelled the tax. Virginia has neither, so the tax on $38,839.97 of taxable income stands with no nonrefundable credit against it."
us,scenario_039,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed AGI correctly at $32,300 but assumed the TCJA sunset took effect in 2026, subtracting a $18,850 standard deduction plus two revived personal exemptions of $10,600 for $29,450 of offsets. Personal exemptions remain repealed in 2026 and the MFJ standard deduction is $32,200, plus $3,300 of aged additional standard deduction and the $12,000 OBBBA senior deduction for two 65+ spouses, totaling $47,500 — more than AGI, so taxable income is $0 rather than the $2,850 it taxed at 10%."
us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model reached the correct $32,300 AGI but then paired a $20,000 MFJ standard deduction with $10,600 of personal exemptions, a combination that exists under no 2026 rule set — personal exemptions are permanently repealed and the 2026 MFJ standard deduction is $32,200. Adding the $3,300 aged additional standard deduction and the $12,000 senior deduction for two spouses aged 65+ wipes out the entire $32,300 of AGI, leaving $0 taxable income instead of the $1,700 it taxed at 10%."
@@ -3586,16 +3688,18 @@ us,scenario_040,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_er
us,scenario_040,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model declared 85% of the $25,200 of Social Security taxable ($21,420) without running the provisional-income tiers: provisional income of $40,600 sits below the $44,000 MFJ second threshold, so only 50% of the $8,600 excess over $32,000, or $4,300, is includible, making AGI $32,300 rather than $49,420. It then treated the senior deduction as a $650 tax offset on $6,500 of income instead of a $6,000 per-spouse deduction ($12,000) and omitted the $3,300 aged additional standard deduction, whereas the full $47,500 of deductions leaves $0 taxable income and $0 tax rather than $1,071."
us,scenario_040,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_040,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model improperly transferred the spouse’s disability to the nondisabled head and invoked Freedom to Work, disabled-adult-child, and spend-down pathways without the categorical facts required for any of them. The head is age 73, nondisabled, receives no SSI, and has income above the applicable aged pathway limit, so the engine assigns Medicaid category NONE."
-us,scenario_040,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly derived $2,142, then overrode its own calculation with an unsupported $204 addition for unspecified state or federal obligations. Arizona imposes no included employee payroll tax here, and the household is far below the Additional Medicare Tax threshold."
-us,scenario_040,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model explicitly calculated $1,736 of Social Security tax and $406 of Medicare tax but submitted $1,948 instead of summing them to $2,142. This is a final arithmetic or output-transcription error."
-us,scenario_040,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct components—$1,736 of Social Security tax and $406 of Medicare tax—but replaced their $2,142 sum with an unexplained $1,550 simplified estimate. Employee-side payroll tax includes both full components on all $28,000 of wages."
-us,scenario_040,payroll_tax,grok-4.3,llm_error,other,False,"The model invoked the correct 7.65% combined employee rate but multiplied or rounded incorrectly: $28,000 × 7.65% equals exactly $2,142, not $2,149."
-us,scenario_040,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract."
+us,scenario_040,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly found $1,530 for Head and $612 for Spouse ($2,142 total) and said Arizona has no employee payroll tax and Additional Medicare Tax does not apply. It then submitted $2,346 to cover unnamed 'additional state obligations' that it had already said don't exist, adding about $204 that has no source."
+us,scenario_040,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model's explanation computes 7.65% FICA on $20,000 and $8,000 as $2,142 and ends with 'value = 2142', but it submitted 2000. Its mention of 'rounding of exemptions' has no basis, because FICA has no exemptions or rounding that would lower the $2,142."
+us,scenario_040,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly computed $1,736 in Social Security tax and $406 in Medicare tax on $28,000 of wages, and correctly ruled out Additional Medicare Tax and any Arizona employee payroll tax. It then submitted 1948 instead of adding the two to get $2,142, which is about $194 short with nothing to justify it."
+us,scenario_040,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model found $1,736 in Social Security tax and $406 in Medicare tax on $28,000 of wages, then cut the total to $1,550 citing 'employer-side exclusions'. Nothing like that reduces employee FICA, which is 7.65% of each spouse's full wages with no offsets, so the correct total is $2,142."
+us,scenario_040,payroll_tax,grok-4.3,llm_error,other,False,"The model used the right method, 7.65% of $28,000 in combined wages, which comes to exactly $2,142. It submitted 2149, an arithmetic slip of $7."
+us,scenario_040,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so nothing could be parsed. The correct value is 7.65% employee FICA on $20,000 and $8,000 of wages, or $2,142."
us,scenario_040,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_040,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified the elderly/disabled gross-income-test exception but then assumed the mortgage interest and property taxes produced an uncapped shelter deduction large enough for eligibility without completing the net-income calculation. After allowable deductions, the household still fails SNAP's net-income test, and the minimum allotment cannot be awarded to an ineligible household."
-us,scenario_040,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model correctly identified that the spouse’s $33,000 bank balance exceeds the stated $2,000 resource limit, then reversed that disqualifying result by invoking home and vehicle exclusions that do not exclude a bank account. Disability and excluded home or vehicle equity do not eliminate the countable-resource test, so the spouse remains outside the aged/disabled pathway."
-us,scenario_040,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model tested only the spouse’s $11,200 receipts against a two-person FPL threshold while asserting that individual income and disregards controlled, instead of applying the engine’s pathway-specific income determination, which yields 2.46 times FPL for MAGI categories and no qualifying aged/disabled category. It also described the 69-year-old spouse as “under 65” and treated aged/disabled categorical status as automatic eligibility."
-us,scenario_040,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model stated that Arizona’s ABD resource limit is $2,000 for an individual or $3,000 for a couple but nevertheless treated the spouse’s $33,000 bank account as compatible with eligibility. A vehicle exclusion does not shelter cash in a bank account, and disability plus low individual wages does not override the resource and income tests; the spouse qualifies through no Medicaid category."
+us,scenario_040,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It ran the 100% FPL aged/disabled income test on the spouse's own $8,000 in wages (after SSI earned-income disregards) and left out the head's $20,000 in wages and $25,200 in Social Security. Those count in the married couple's test, which puts household income at 2.46x FPL. It also admitted the $33,000 in bank assets exceeded the limit, then wrongly assumed home and vehicle exclusions would fix that, even though bank deposits are countable."
+us,scenario_040,spouse_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated being disabled as enough on its own for Arizona's aged/disabled Medicaid and never applied that category's 100% FPL income limit. The couple's income is 2.46x FPL, so the spouse fails the income test and fits no pathway."
+us,scenario_040,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It correctly named the 100% FPL limit for the aged/disabled pathway and even added up the household's $56,400 total. It then claimed PolicyEngine judges only the spouse's own income and declared eligibility, when the test uses the couple's combined wages and Social Security, about 2.46x the two-person FPL. That is well over the limit."
+us,scenario_040,spouse_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It tested only the spouse's $8,000 in wages against the aged/disabled FPL limit and left out the head's $20,000 in wages and $25,200 in Social Security. Those count for a married couple and push household income to 2.46x FPL, against a 100% FPL limit. It also noted the $33,000 in bank assets was above the $2,000/$3,000 resource limit and then set that aside, relying on an SSI-linked status the spouse does not have, since SSI received is $0."
us,scenario_040,spouse_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model reversed the Medicare age comparison by treating age 69 as under 65. Because the spouse is already older than 65, Medicare eligibility follows directly from age, and the model’s SSDI-duration analysis is irrelevant."
us,scenario_040,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model reached the correct result mid-reasoning — Arizona subtracts the $4,300 of taxable Social Security, leaving $28,000 of wages against an Arizona married-filing-jointly standard deduction of roughly $32,200 for 2026, so taxable income is zero — and then discarded it by inventing a smaller deduction and a residual base of $5,920 ($148 ÷ 0.025). Arizona's standard deduction conforms to the full federal MFJ amount, which exceeds $28,000 by over $4,000, so no residual base exists and the 2.5% flat tax applies to nothing."
us,scenario_040,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model used a $20,000 Arizona married-filing-jointly standard deduction 'with both spouses 65+', but Arizona conforms to the base federal standard deduction (about $32,200 for MFJ in 2026) and grants no additional age-65 amount at all — the deduction is neither $20,000 nor age-adjusted. With the correct $32,200 deduction, the $28,000 Arizona AGI produces zero taxable income instead of the $8,000 base the model taxed at 2.5%."
@@ -3605,102 +3709,104 @@ us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error,
us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed $21,420 of Social Security as part of the Arizona base, missing both that provisional income of $40,600 puts only $4,300 of the $25,200 in federal AGI and that Arizona subtracts 100% of the Social Security benefits included in federal AGI. It also contradicted its own arithmetic, computing $484.25 and then submitting $486.25; with Social Security removed, Arizona AGI is $28,000, below the roughly $32,200 MFJ standard deduction, so taxable income and tax are both zero."
us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0."
us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left the $928 of qualified dividends out of gross income (non-SS income $32,807 instead of $33,735). That understated both provisional income and taxable Social Security ($16,057 instead of $16,846.25), and it cut taxable income to $24,714 instead of $26,430.71. It also counted the $71 BDC income as preferential income and did loose bracket arithmetic, which is how it reached $2,133."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and used the $32,200 joint standard deduction with joint brackets. QSS status requires a dependent child, so this filer is Single, with a $16,100 + $2,050 deduction and a 10% bracket ending at $12,400. It also taxed the $3,753 of capital gains as ordinary income instead of at the 0% preferential rate."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It put the entire $29,580 of Social Security into income without applying the provisional-income formula, which makes only $16,846.25 taxable. It used an invented $28,900 surviving-spouse deduction and left out the $6,000 senior deduction. It then taxed the $4,681 of gains at 15% instead of 0% and subtracted a nonexistent 'dependent exemption credit'."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security dependent benefits as belonging to someone else and excluded them. In fact $16,846.25 is taxable to the head once provisional income of $48,525 is run through the 25k/34k formula. It also left out the $928 qualified dividends and the $6,000 senior deduction, which produced a $14,657 taxable income unrelated to the correct $26,430.71."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used Qualifying Surviving Spouse parameters (a $30,700 deduction and joint brackets) even though there is no dependent child, so the filer is Single. It also made 85% of all benefits ($25,143) taxable instead of the $16,846.25 the two-tier formula gives, and then applied an unexplained downward 'adjustment' to land on $1,245."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It made 85% of all benefits ($25,143) taxable instead of applying the formula min(85% of benefits, 0.85 × (48,525 − 34,000) + 4,500) = $16,846.25. That inflated AGI to about $58,878 instead of $50,580.71. It then produced $2,699 through loose arithmetic instead of computing tax on $21,749.71 of ordinary taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with joint brackets and the $32,200 + $1,650 joint deduction. With no dependent child QSS is unavailable, so the Single $16,100 + $2,050 deduction and the Single brackets apply. That leaves $21,749.71 of ordinary income taxed at 10%/12% rather than $6,036 at 10%."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $17,750 deduction and omitted the $6,000 OBBBA senior deduction for filers 65 and older, so taxable income came out about $6,000 too high. It also added the $71 BDC income on top of dividends and used outdated bracket figures."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It excluded the $29,580 of Social Security on the theory that dependent benefits are not the taxpayer's, but $16,846.25 of them is taxable to the head. It then reported $0 even though its own $16,792 taxable income produces positive tax at 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with the $32,200 joint deduction and joint brackets. With no dependent child the filer is Single, entitled to $16,100 + $2,050 + $6,000. That leaves $21,749.71 of ordinary income taxed at 10%/12% instead of $6,050 at 10%."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $17,650 standard deduction and omitted the $6,000 senior deduction for filers 65 and older, which overstated taxable income by about $6,000. It also counted the $71 BDC income as extra income, which raised taxable SS to $16,907."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied only a $17,425 standard deduction and never took the $6,000 senior deduction, so taxable income was $33,287.60 instead of $26,430.71. It also counted the $71 BDC income on top of dividends."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law ($8,600 deduction, a $5,500 personal exemption, and the 15% bracket), but OBBBA made the TCJA parameters permanent. The correct 2026 figures are a $16,100 + $2,050 deduction plus the $6,000 senior deduction, with 10%/12% brackets."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used a $17,350 deduction with no $6,000 senior deduction, which produced $33,362.60 of taxable income instead of $26,430.71. Its SS amount of $16,907 also includes the $71 BDC income double-counted on top of dividends."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and used a $10,300 deduction, a $5,200 personal exemption, and the 15% bracket. OBBBA kept TCJA's $16,100 + $2,050 deduction and 10%/12% brackets, and it added the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used head-of-household status even though there is no qualifying person, and its $3,156 is roughly $794 above the correct tax. That gap matches omitting the $6,000 senior deduction at the 12% marginal rate, which adds about $720."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It had the correct $50,581 AGI but subtracted only the elderly Single standard deduction and skipped the $6,000 senior deduction. That leaves $32,431 of taxable income instead of $26,430.71, and it rounded up further to $3,500."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA law with a combined $15,150 deduction and exemption and the 15% bracket. OBBBA made the $16,100 + $2,050 deduction and the 10%/12% brackets permanent and added the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $1,240 + 12% × $9,349.71 = $2,361.96 on $21,749.71 of ordinary taxable income. Its $2,878 is about $516 higher, roughly the effect of taxing the $4,681 of qualified dividends and capital gain at 12% instead of the 0% rate."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $31,550 surviving-spouse deduction, but with no dependent child the filer is Single. It also understated taxable Social Security at $9,846 instead of $16,846.25 and omitted the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the Single age-65 standard deduction from the $50,581 AGI and omitted the $6,000 senior deduction. The submitted $3,090 matches tax on $32,431 of taxable income, and its explanation contradicts that value by citing $3,584."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and never applied the $6,000 senior deduction. Its $3,734 also exceeds the roughly $3,082 that the no-senior-deduction path gives, which is consistent with taxing the $4,681 of preferential income at ordinary rates."
-us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $17,850 deduction with no $6,000 senior deduction. It also made a $1,000 addition error (non-SS income $34,806 instead of $33,805 including BDC), which inflated taxable SS to $17,757 and AGI to $52,563."
-us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It returned the placeholder figure $1,234.56 with no computation. Its premise that Social Security is 'largely nontaxable' is wrong, because provisional income of $48,525 makes $16,846.25 taxable and leaves $26,430.71 of taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age deductions fully offset income. In fact $33,735 of other income plus $16,846.25 of taxable Social Security exceeds the $24,150 of deductions by $26,430.71, of which $21,749.71 is ordinary income taxed at 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits eliminate all tax, but AGI of $50,580.71 minus $24,150 of deductions leaves $26,430.71 taxable. No nonrefundable credit applies, because the elderly credit is phased out at this AGI."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $33,200 surviving-spouse deduction and taxed the remainder at 10% under joint brackets. QSS requires a dependent child, so Single parameters apply ($16,100 + $2,050 + $6,000 senior deduction, with the 10% bracket ending at $12,400), and it also never took the senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $58,949 AGI treats 85% of all benefits as taxable instead of the $16,846.25 the provisional-income formula gives. It also taxed qualified dividends and capital gains at 15%, although they sit entirely within the 0% band at $26,430.71 of taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the qualifying-surviving-spouse deduction and joint brackets. With no dependent child the filer is Single, which leaves $21,749.71 of ordinary taxable income rather than $6,036."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) standard deduction and kept all ordinary income in the 10% bracket. The filer has no dependent child and must use Single parameters, under which $9,349.71 of ordinary income falls in the 12% bracket."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status with the $33,850 joint age-adjusted deduction. QSS requires a dependent child, so the Single $18,150 deduction applies, and ordinary taxable income is $21,749.71 rather than $6,036.05."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $34,785 taxable income after $24,150 of deductions implies roughly $25,143 of taxable SS (85% of all benefits) instead of $16,846.25. It also gave 0% treatment only to the $928 qualified dividends and taxed the $3,753 of capital gains as ordinary income."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It added the $71 qualified BDC income as extra ordinary income on top of dividends, which also raised taxable Social Security by 85% of that amount. Together that put about $131 more in ordinary taxable income, and 12% of it is the roughly $16 overstatement."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used surviving-spouse (joint) deductions, although a filer with no dependent child is Single. It also wrongly concluded that only preferential income remains; even under its own QSS assumption about $6,036 of ordinary income is left taxable."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and omitted the $6,000 senior deduction, and it cited an inapplicable mortgage-interest deduction. Its $3,125 matches tax on roughly $32,400 of taxable income instead of $26,430.71."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law ($10,507 deduction, $5,387 exemption, and the 15% bracket). OBBBA made the TCJA $16,100 + $2,050 deduction and 10%/12% brackets permanent and added the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used 'post-TCJA' expired-law parameters ($8,300 deduction, $5,300 personal exemption, and the 15% bracket). The 2026 law keeps the $16,100 + $2,050 deduction and 10%/12% brackets and adds the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It assumed TCJA sunset and used an $8,300 deduction, a $5,300 exemption, and 10%/15% brackets. OBBBA made TCJA permanent, so the deductions total $24,150 including the $6,000 senior deduction and the ordinary rates are 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied reverted pre-TCJA parameters ($10,233 deduction and the 15% bracket) and omitted the $6,000 senior deduction. It miscomputed taxable SS as $24,205 instead of $16,846.25 and taxed the $3,753 of capital gains as ordinary income."
-us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It deducted only the $18,150 standard deduction and never took the $6,000 senior deduction, so taxable income came out at $30,478. It also double-counted the farm loss (−$2,254 instead of −$1,127), which understated taxable SS at $15,949."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It had the correct $48,525 provisional income and $24,150 of deductions but computed taxable Social Security as $24,108.75. The formula gives 0.85 × ($48,525 − $34,000) + $4,500 = $16,846.25, so its taxable income was overstated by $7,262.50."
-us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used joint (MFJ/QSS) $32k/$44k Social Security thresholds and a $32,150 surviving-spouse deduction. A widow with no dependent child is Single, with $25k/$34k thresholds, $16,846.25 of taxable SS, and $24,150 of deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with MFJ rates and a $32,200 base deduction, and used joint SS thresholds that produced $9,907 of taxable SS. Single status gives $16,846.25 of taxable SS and $18,150 plus $6,000 of deductions, and it also taxed the $3,753 of capital gains as ordinary income."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used QSS brackets and deduction even though there is no dependent child. It also added the $4,700 tax-exempt IRA distributions to provisional income, computed $25,698 of taxable SS (above the 85% cap), and omitted the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no computation, and $21,824 is about nine times the correct tax. The correct tax comes from $21,749.71 of ordinary taxable income at 10%/12% with preferential income at 0%, whereas its figure implies taxing nearly all gross income at high marginal rates."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left the $928 of qualified dividends out of its non-Social Security income ($32,807 instead of $33,735). That understated provisional income, taxable Social Security ($16,057 instead of $16,846) and AGI ($48,864 instead of $50,581). Taxable income therefore fell to $24,714 instead of $26,431, and ordinary tax came out low at $2,133."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and used the $32,200 joint standard deduction, the $1,650 married aged add-on and joint brackets. QSS requires a dependent child, so the head files Single with a $16,100 + $2,050 + $6,000 deduction. It also taxed the $3,753 of capital gains as ordinary income instead of putting them in the 0% net-capital-gain bucket."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted the full $29,580 of Social Security in income instead of the $16,846 taxable under the §86 formula, used a made-up $28,900 surviving-spouse deduction, and left out the $6,000 senior deduction. It also taxed $4,681 of qualified dividends and gains at 15% even though they fall in the 0% band, and then subtracted a nonexistent 'dependent exemption credit' of $783."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the deductions wiped out taxable income without doing the computation. AGI is $50,581 against $24,150 of total deductions, which leaves $26,431 of taxable income. $21,750 of that is ordinary income taxed at 10%/12% for $2,362, not zero."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security dependent benefits as belonging to someone else and not taxable to the head, which dropped the $16,846 of taxable Social Security from AGI. It also left out the $928 of qualified dividends and the $6,000 OBBBA senior deduction. These offsetting errors produced $1,359."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a Qualifying Surviving Spouse standard deduction ($30,700 + $1,650) and joint brackets even though there is no dependent child, so the head files Single. It also set taxable Social Security at the 85% cap ($25,143) instead of the formula amount of $16,846, left out the $6,000 senior deduction, and then cut its own $2,603 figure to $1,245 with no computation."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It took 85% of Social Security ($25,143) as taxable instead of applying the lesser-of formula: 0.85×($48,525−$34,000)+$4,500 = $16,846. That inflated AGI to about $58,878. Its final $2,699 also does not follow from the $34,700 of taxable income it stated."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and applied the $32,200 joint standard deduction, the $1,650 aged add-on and joint brackets. With no dependent child, the head files Single, so the base deduction is $16,100 + $2,050 and $21,750 of ordinary income is taxed at 10%/12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction for taxpayers 65 and older and used a guessed $17,750 standard deduction instead of $18,150. It also added the $71 of BDC income to gross income. Taxable income came out around $32,949 instead of $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as not taxable to the head, which dropped $16,846 of taxable Social Security. It then reported $0 even though its own figure of about $16,792 of taxable income would produce several hundred dollars of 10%/12% tax."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with the $32,200 joint standard deduction, the $1,650 add-on and joint brackets. Without a dependent child the head files Single with $24,150 of total deductions, which leaves $21,750 of ordinary income at 10%/12% instead of $6,050 at 10%."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and used 2025-level amounts ($15,650 + $2,000) instead of the 2026 $16,100 + $2,050. That overstated taxable income by about $6,500, to $33,063 instead of $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It deducted only a $17,425 standard deduction, leaving out the $6,000 senior deduction and understating the 2026 base plus aged amount ($18,150). Ordinary taxable income came out at $28,536 instead of $21,750."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 sunset law, with an $8,600 standard deduction, a $5,500 personal exemption and a 15% bracket. OBBBA made the TCJA schedule permanent, so the correct deductions are $16,100 + $2,050 + the $6,000 senior deduction with 10%/12% brackets and no exemption."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used a $17,350 standard deduction and left out the $6,000 OBBBA senior deduction. It also understated the 2026 single-plus-aged amount ($18,150), so ordinary taxable income was $28,682 instead of $21,750."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, using a $10,300 deduction, a $5,200 personal exemption and a 15% bracket. Under OBBBA's permanent extension, the 2026 single filer gets $16,100 + $2,050 + $6,000 in deductions, with 10%/12% brackets."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used head-of-household status even though the head has no qualifying person. Its $3,156 is consistent with about $33,000 of taxable income, meaning it left out the $6,000 senior deduction that brings taxable income down to $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It got AGI right ($50,581) but subtracted only the elderly single standard deduction, leaving out the $6,000 OBBBA senior deduction. The $3,500 result implies roughly $31,000 taxed at ordinary rates instead of $21,750."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 brackets and personal exemptions ($15,150 of combined deductions). OBBBA made the TCJA parameters permanent, so the correct deductions are $24,150 and the brackets are 10%/12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path taxes $21,750 of ordinary income at 10%/12% ($2,362) and $4,681 of qualified dividends and gains at 0%. The $2,878 answer is consistent with taxing roughly $4,300 more as ordinary income, such as the qualified dividends and gains taxed at 12% instead of the 0% rate."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse, using the $32k/$44k joint Social Security thresholds (taxable Social Security $9,846) and a $31,550 joint-style standard deduction. With no dependent child the head files Single, so taxable Social Security is $16,846 and total deductions are $24,150."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the elderly single standard deduction from $50,581 of AGI and left out the $6,000 OBBBA senior deduction. Its explanation also states $3,584, which does not match the submitted $3,090."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and left out the $6,000 senior deduction. The $3,734 implies about $33,000 taxed at 10%/12%, against the correct $21,750 of ordinary taxable income."
+us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction (deducting only $17,850). It also mis-added non-Social Security income as $34,806 instead of $33,806, which inflated taxable Social Security to $17,757 and AGI to $52,563."
+us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It gave no computation and called the Social Security 'largely nontaxable'. In fact $16,846 of the $29,580 is taxable, which leaves $26,431 of taxable income and $2,362 of tax. Its $1,234.56 is a placeholder-style figure."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that the standard and age-related deductions fully offset income. AGI of $50,581 less $24,150 of deductions leaves $26,431 of taxable income, including $21,750 of ordinary income taxed at 10%/12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits brought tax to zero. No nonrefundable credit applies, and $26,431 of taxable income remains after the $24,150 of deductions, producing $2,362 of tax."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $33,200 surviving-spouse standard deduction with joint brackets. With no dependent child the head files Single, so deductions total $16,100 + $2,050 + $6,000 = $24,150. It also left out the senior deduction it would otherwise have claimed."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It made 85% of Social Security taxable, pushing AGI to $58,949 instead of $50,581. It also taxed the qualified dividends and capital gains at 15%, even though total taxable income of $26,431 keeps all $4,681 in the 0% band."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the qualifying-surviving-spouse standard deduction and joint brackets. With no dependent child, the head files Single with an $18,150 base-plus-aged deduction, which leaves $21,750 of ordinary income instead of $6,036."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) standard deduction, which put all ordinary income in the 10% bracket. As a Single filer, the head has $21,750 of ordinary income taxed at 10% up to $12,400 and 12% above."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse with a $33,850 age-adjusted joint standard deduction. With no dependent child, the head files Single with $16,100 + $2,050 + $6,000, leaving $21,750 of ordinary taxable income instead of $6,036."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $34,785 of taxable income implies about $25,000 of taxable Social Security, the 85% cap, instead of the formula amount of $16,846. It also put only the $928 of qualified dividends at 0% and taxed the $3,753 of capital gains as ordinary income."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It added the $71 of qualified BDC income as separate gross income on top of the dividends. That raised provisional income and taxable Social Security by $60.35, so AGI went up by about $131 and tax by about $16 ($2,377.79 instead of $2,361.96)."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used surviving-spouse (joint) deductions and concluded that only 0%-rate preferential income remained. As a Single filer with $24,150 of deductions, the head still has $21,750 of ordinary taxable income."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction (plus QBI and mortgage interest) and left out the $6,000 OBBBA senior deduction for those 65 and older. The $3,125 implies about $28,100 of ordinary taxable income instead of $21,750."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rules: a $10,507 standard deduction, a $5,387 personal exemption and 10%/15% brackets. OBBBA kept the TCJA schedule, so the correct figures are a $16,100 + $2,050 standard deduction, the $6,000 senior deduction and 10%/12% brackets."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration parameters: an $8,300 standard deduction, a $5,300 exemption and a 15% bracket. 2026 law under OBBBA provides $16,100 + $2,050 + a $6,000 senior deduction with 10%/12% brackets."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the pre-TCJA 2026 sunset schedule ($8,300 standard deduction, $2,000 aged amount, $5,300 personal exemption, 15% bracket). OBBBA made the TCJA rates and the $16,100 standard deduction permanent and added the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used reverted pre-TCJA parameters (a $10,233 deduction and a 15% bracket). It also overstated taxable Social Security at $24,205 instead of $16,846 and taxed the $3,753 of capital gains as ordinary income, which brought taxable income to $47,778 instead of $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and deducted only $18,150. It also counted the farm loss twice (−$2,254), even though 'farm income' and 'farm operations income' describe the same −$1,127 item, which understated taxable Social Security and AGI."
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer or explanation for this variable, so there was no value to score."
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed provisional income correctly at $48,525 but then reported $24,108.75 of taxable Social Security. The single-filer §86 formula gives 0.85×($48,525−$34,000)+$4,500 = $16,846.25, so AGI was overstated by $7,262.50."
+us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the joint $32k/$44k Social Security thresholds and a $32,150 surviving-spouse standard deduction. With no dependent child the head files Single, so taxable Social Security is $16,846 and deductions are $18,150 plus the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with MFJ rates, joint Social Security thresholds and a $32,200 standard deduction. Single status applies because there is no dependent child. It also treated the $3,753 of gains as short-term ordinary income instead of 0%-rate net capital gain."
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse with no dependent child and used a $31,675 deduction and joint brackets, leaving out the $6,000 senior deduction. It also added the $4,700 of tax-exempt IRA distributions to provisional income and reported $25,698 of taxable Social Security, which exceeds the 85% cap of $25,143."
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $21,824 is about 43% of the $50,581 AGI. The correct computation taxes $21,750 of ordinary income at 10%/12% for $2,362."
us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it."
us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base.
us,scenario_042,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"The model's own reasoning cut WI income to about $9,700 and said the standard deduction fully covers it, yet it submitted $953 without any computation connecting the two. The engine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI. The $12,671.36 standard deduction and $950 of exemptions bring that to $16,360.34 taxable and $584.74 of tax, and the model never applied the $300 school property tax credit that follows."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It gave the surviving spouse the married-joint sliding-scale standard deduction (about $25,728). Wisconsin has no surviving-spouse status, so the engine uses the single deduction of $12,671.36 on $29,981.71 of WI AGI, which leaves $16,360.34 taxable and $584.74 of tax. It also said property taxes earn no credit, which leaves out the $300 school property tax credit (12% of the first $2,500 of property tax)."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It left the full $29,580 of Social Security in Wisconsin taxable income (about $55,000), although Wisconsin subtracts all $13,655.95 of taxable Social Security. It then wiped out the resulting ~$2,500 tax with a homestead credit. The homestead credit is refundable and income-limited, so the only nonrefundable offset is the $300 school property tax credit against $584.74 of tax."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement income subtraction, which Wisconsin allows only when WI AGI is under $15,000 for a single filer, and treated the $700/$250 exemptions as $20 credits instead of deductions. Its own figures ($595 − $300 − $40 = $255) also do not reach its reported $0. The engine taxes $16,360.34 for $584.74 and subtracts the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that is barred once WI AGI exceeds $15,000 (single) and used a ~$9,200 standard deduction instead of $12,671.36. It then dropped its own $731 tax estimate to zero by citing unspecified real-estate and mortgage credit effects. The only nonrefundable credit here is the $300 school property tax credit, which takes $584.74 of tax to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It used a sliding-scale standard deduction of only about $5,000 and kept the $3,753 of capital gains in WI income, which produced about $27,500 of taxable income. At WI AGI of $29,981.71, the single standard deduction is $12,671.36, and with $950 of exemptions that leaves $16,360.34 taxable and $584.74 of tax before the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67+ retirement income exclusion to the IRA and pension income, which dropped Wisconsin income to about $8,600 and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security. That leaves $29,981.71 of WI AGI, $16,360.34 of taxable income, $584.74 of tax, and $284.74 after the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a married-joint Wisconsin filer, which gave a $10,000 retirement subtraction and a ~$21,151 joint standard deduction. Wisconsin files this person as single: the $5,000 retirement subtraction is barred above $15,000 of WI AGI, and the single deduction is $12,671.36. It also treated the $700 exemption as a tax credit, which zeroed a tax the engine computes as $584.74 less the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It computed about $790 of tax and then erased it with an itemized deduction credit on real estate taxes. Wisconsin's itemized deduction credit excludes property taxes, and the $350 of medical expenses produces no credit, so the only nonrefundable credit is the flat $300 school property tax credit. It also kept the capital gains in WI AGI, which overstates the base above the engine's $29,981.71."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gain distributions in WI AGI ($33,735, versus the engine's $29,981.71). It also never subtracted the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes). Its $790 is therefore a pre-credit tax on an overstated base."
-us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It started from $33,806 of WI income, which includes $3,824 of capital gain and BDC income that is absent from the engine's $29,981.71 of WI AGI. It used a lumped ~$14,800 deduction instead of the $12,671.36 standard deduction plus $950 of exemptions, and it never claimed the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It applied a $13,410 standard deduction with no phase-down to $33,806 of WI income and skipped the $700 + $250 exemptions. It used 3.54%/4.65% rates instead of 3.5%/4.4%. It also said no nonrefundable credits apply, which leaves out the $300 school property tax credit that the engine subtracts from $584.74."
-us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement income subtraction that Wisconsin allows only when WI AGI is below $15,000 (single) and left out the $250 age-65 exemption. It also never applied the $300 school property tax credit, so its $534 replaces the correct $584.74 − $300."
-us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,837 standard deduction on $33,806 of WI income and took no $700/$250 exemptions. That puts taxable income at $24,969, versus the engine's $16,360.34, and it also left out the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Its $29,446 of taxable income leaves the $12,671.36 sliding-scale standard deduction and $950 of exemptions essentially unapplied, which yields about $1,170 of tax against the engine's $584.74 on $16,360.34. It also subtracted a nonexistent $50 personal exemption credit alongside the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,137 matches taxing about $29,000 of Wisconsin income at 3.5%/4.4% with no standard deduction, no exemptions, and no school property tax credit. The engine reduces $29,981.71 of WI AGI by $12,671.36 and $950 to $16,360.34, taxes that at $584.74, and subtracts the $300 credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It applied only the Social Security subtraction and a standard deduction. Its $860 matches about $22,600 of taxable income, which means no $950 of exemptions and a base that still includes the capital gains. It also left out the $300 school property tax credit that brings the engine's $584.74 down to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It reached $25,235 of taxable income by taking only about $8,571 of deductions from $33,806 of Wisconsin income. The engine's WI AGI is $29,981.71, and it subtracts a $12,671.36 standard deduction plus $950 of exemptions to get $16,360.34 taxable. The model applied the $300 school property tax credit correctly, but against an overstated $981 of tax."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,973 matches taxing about $48,000 at 3.5%/4.4%, which is roughly the full federal AGI with taxable Social Security left in and no deduction. Wisconsin subtracts the $13,655.95 of taxable Social Security, the $12,671.36 standard deduction, and $950 of exemptions, leaving $16,360.34 taxed at $584.74 before the $300 credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gain distributions in WI AGI ($33,735, versus the engine's $29,981.71) and never applied the $300 school property tax credit. Its $786 is therefore a tax before that credit, on an overstated base."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation states $1,617 while it submitted $1,348, and both figures reflect a tax base far above the engine's $16,360.34 of taxable income. It also relied on a nonexistent 'senior credit'. The engine's $584.74 of tax less the $300 school property tax credit is $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,428 after a $300 school property tax credit implies about $1,728 of tax on roughly $42,000 of taxable income, which matches leaving taxable Social Security in Wisconsin income or skipping the standard deduction. Wisconsin subtracts the $13,655.95 of taxable Social Security, then the $12,671.36 deduction and $950 of exemptions, leaving $16,360.34 taxable."
-us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It subtracted all $24,086 of IRA and pension income as senior retirement income, which cut WI AGI to about $9,720 and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security and keeps the IRA and pension income in $29,981.71 of WI AGI. That produces $584.74 of tax, less the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It returned a placeholder-style $123.45 with no computation behind it. The engine takes $29,981.71 of WI AGI, subtracts the $12,671.36 standard deduction and $950 of exemptions to get $16,360.34 taxable, computes $584.74 of tax, and subtracts the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It zeroed the tax by applying pension and retirement exclusions to the IRA and pension income. PolicyEngine applies no retirement exclusion here: it taxes $29,981.71 of WI AGI, less the $12,671.36 deduction and $950 of exemptions, at $584.74 and then subtracts the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It claimed the retirement income is fully offset without naming any exclusion or deduction that does so. The standard deduction ($12,671.36) and exemptions ($950) absorb only $13,621.36 of the $29,981.71 of WI AGI, which leaves $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It claimed the school property tax credit on $5,011 of real estate taxes fully offsets the tentative tax. That credit is capped at $300 (12% of the first $2,500 of property tax), while the tentative tax is $584.74 on $16,360.34 of taxable income, which leaves $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It said no nonrefundable state credit applies, which leaves out the $300 school property tax credit on the $5,011 of real estate taxes. It also taxed Wisconsin income that still includes the capital gain distributions, so its ~$800 is a tax before that credit on a base above the engine's $16,360.34."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67+ retirement-income subtraction to the taxable IRA and pension income, which cut Wisconsin income below the deductions and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI, $584.74 of tax, and $284.74 after the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Its explanation uses only the Social Security exclusion, deductions, and exemptions, with no school property tax credit. Its $430 is therefore a tentative tax below the engine's $584.74 on $16,360.34 of taxable income, and it skips the required $300 school property tax credit that lowers the tax to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67 retirement-income subtraction to the IRA and pension income and reported $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, taxes $16,360.34 (after the $12,671.36 deduction and $950 of exemptions) at $584.74, and subtracts the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It stopped at a tax before credits and never applied the $300 school property tax credit on the $5,011 of real estate taxes. Its base still includes the capital gain distributions, so its $770 is above the engine's $584.74 even before that credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It applied an age-qualified retirement-income subtraction to the $24,086 of IRA and pension income, which let the standard deduction absorb the remainder and gave $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI and $284.74 of tax after the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It used a retirement deduction on the IRA and pension income, together with the standard deduction and credits, to eliminate the tax. PolicyEngine applies no retirement subtraction here: it taxes $16,360.34 at $584.74 and subtracts only the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It eliminated the tax with retirement exclusions on the IRA and pension income. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, so $29,981.71 of WI AGI remains, and after the $12,671.36 deduction, $950 of exemptions, and the $300 school property tax credit the tax is $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gains (after a 30% exclusion) and the $71 of BDC income in WI AGI ($32,680, versus $29,981.71) and used a $10,683 standard deduction instead of $12,671.36. It also set the school property tax credit at $366, although the credit is capped at $300 (12% of the first $2,500 of property tax)."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"Its $32,680 of WI AGI keeps the non-Schedule-D capital gains (after a 30% exclusion) and the BDC income, which are not in the engine's $43,637.65 federal AGI. The engine's WI AGI is $29,981.71, which with the $12,671.36 deduction and $950 of exemptions gives $16,360.34 taxable and $584.74 of tax before the $300 credit that the model applied correctly."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It built federal AGI of $50,713 with the capital gain distributions included and carried $32,680 of WI income forward, against the engine's $43,637.65 of AGI and $29,981.71 of WI AGI. That raised taxable income to $19,408, versus $16,360.34, and tax to $719, versus $584.74, before the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It applied a $13,900 standard deduction with no phase-down, took no $950 of exemptions, and used 3.54%/4.65% rates instead of 3.5%/4.4%, all on $33,806 of WI AGI that includes the capital gains. It also left out the $300 school property tax credit that reduces the engine's $584.74 to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It overstated WI income at $32,679 by including the capital gain distributions, used an $11,351 deduction instead of $12,671.36, and taxed the second bracket at 4.65% instead of 4.4%. That produced about $779 of tax before the $300 credit, versus the engine's $584.74 on $16,360.34."
-us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the engine's $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It computed $24,108.75 of taxable Social Security from an inflated $57,843.75 federal AGI and kept the $3,753 of capital gains (less 30%) in WI income ($32,609.10). The engine's taxable Social Security is $13,655.95 and its WI AGI is $29,981.71, so taxable income is $16,360.34 and tax is $584.74 before the $300 credit."
-us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It excluded $24,000 of IRA and pension income as senior retirement income and also used a $24,000 surviving-spouse standard deduction, which drove the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security and applies the single deduction of $12,671.36 plus $950 of exemptions, which yields $584.74 of tax and $284.74 after the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a married-joint filer for the standard deduction, skipped the $950 of exemptions, and kept the capital gains in WI AGI ($33,806). It then taxed $20,506 entirely at 3.5%, although the first bracket ends near $15,000. It also invented an income phase-out that cut the school property tax credit to $276 instead of the flat $300 (12% of the first $2,500)."
-us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Its arithmetic is incoherent: it computed $297 of tax before credits, subtracted roughly $339 of credits, and reported a higher $417.88. It also used a head-of-household deduction and a nonexistent 'personal tax credit'. The engine taxes $16,360.34 for $584.74 and subtracts only the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,830 matches about $44,000 of taxable income at 3.5%/4.4%, which is taxing roughly the full federal AGI without subtracting taxable Social Security or applying deductions. The engine subtracts the $13,655.95 of taxable Social Security, the $12,671.36 deduction, and $950 of exemptions, then the $300 school property tax credit, for $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"Its own steps applied a $24,000 retirement exclusion that cut Wisconsin income to about $9,700, and it said the standard deduction fully covered that income, which means zero tax. It still submitted $953, a number that follows from neither its own computation nor the reference path (WI AGI $29,981.71, $16,360.34 taxable, $584.74 tax, less the $300 school property tax credit)."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a joint filer for Wisconsin and used the married-joint sliding standard deduction (~$25,728) instead of the single-schedule $12,671.36, which cut taxable income to ~$7,057. It also set nonrefundable credits to zero and missed the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes)."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It left the $29,580 of Social Security in Wisconsin income and used a ~$9,520 standard deduction, producing ~$55,000 of taxable income, then cancelled the tax with a homestead credit. Wisconsin subtracts all taxable Social Security (WI AGI $29,981.71). The homestead credit is refundable and not available at this income; the only nonrefundable property-tax credit is the $300 school property tax credit, applied against $584.74 of tax."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"It reached $0 by excluding the IRA and pension income under a Wisconsin senior retirement exclusion. The reference subtracts only the $13,655.95 of taxable Social Security and no retirement income, leaving WI AGI of $29,981.71, $16,360.34 taxable, and $284.74 after the $300 school property tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that Wisconsin only allows when federal AGI is under $15,000 for a single filer, used a ~$11,790 qualifying-widow standard deduction, and treated the exemptions as $20 credits. Even on its own numbers, $595 minus the $300 school property tax credit and $40 is $255, not $0; the reference nets $584.74 − $300 = $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It computed a ~$731 tentative tax, then claimed without any arithmetic that real-estate and mortgage 'credit interactions' bring it to $0. Property taxes do not count toward Wisconsin's itemized deduction credit and no mortgage interest was reported, so the only nonrefundable offset is the $300 school property tax credit, which the reference subtracts from $584.74 to get $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It used a Wisconsin standard deduction of only ~$5,000, leaving ~$27,500 taxable. The single sliding-scale deduction at the reference's $29,981.71 WI AGI is $12,671.36, which with $950 of exemptions leaves $16,360.34 taxable and $584.74 of tax before the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It subtracted $24,000 of IRA and pension income under Wisconsin's age-67+ retirement income exclusion, plus a 30% capital-gain exclusion, which dropped Wisconsin income to ~$8,600 and the tax to $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, so WI AGI is $29,981.71, tax is $584.74, and the result is $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It invented a $10,000 surviving-spouse retirement subtraction, treated the surviving spouse as a joint filer with a ~$21,151 standard deduction, and applied the $700 exemption as a $700 tax credit. In the reference this filer uses the single sliding standard deduction ($12,671.36), and the $700 + $250 exemptions reduce income rather than tax; the $5,000 retirement subtraction is not available above $15,000 of federal AGI. That leaves $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It computed ~$790 of tax, then claimed the itemized deduction credit on real estate taxes and medical costs wipes it out. Property taxes do not count toward Wisconsin's itemized deduction credit and $350 of medical costs is below the floor, so the only nonrefundable credit is the $300 school property tax credit, which leaves a positive tax ($284.74 in the reference)."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It followed the right structure (subtract taxable Social Security, sliding standard deduction, $950 exemptions, 3.5%/4.4% brackets) but stopped at the $790 tentative tax and never subtracted the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes). Its WI income of $33,735 also includes the $3,753 of capital gains, compared with the reference's $29,981.71."
+us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It used a ~$14,800 standard deduction with an invented 'elderly addition' in place of the phased single deduction plus the $700 + $250 exemptions. It applied no nonrefundable credit, missing the $300 school property tax credit that the reference subtracts from $584.74."
+us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It used the pre-2023 Wisconsin rates of 3.54%/4.65%, took an unphased $13,410 standard deduction, skipped the $700 + $250 exemptions, and said no nonrefundable credits apply, missing the $300 school property tax credit. The reference taxes $16,360.34 at 3.5%/4.4% for $584.74, then subtracts the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that Wisconsin only allows when federal AGI is under $15,000 for a single filer, dropped the $250 age-65 exemption, and never applied the $300 school property tax credit. The reference subtracts only taxable Social Security (WI AGI $29,981.71), uses $950 of exemptions, and nets $584.74 − $300 = $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,837 standard deduction, far below the single sliding-scale amount of $12,671.36, left out the $950 of personal and age exemptions, and applied no school property tax credit. That gave $24,969 taxable and $967 of tax, against the reference's $16,360.34 taxable and $284.74 net."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Its Wisconsin taxable income of ~$29,446 is essentially Wisconsin AGI with neither the $12,671.36 sliding standard deduction nor the $950 of exemptions taken out, and it then subtracted an invented $50 personal exemption credit. The reference reduces WI AGI of $29,981.71 to $16,360.34 taxable before applying the 3.5%/4.4% rates and the $300 school property tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,137 matches taxing about $28,900 at 3.5%/4.4% with essentially no standard deduction or exemptions and no $300 credit. In the reference, the $12,671.36 sliding deduction and $950 of exemptions reduce WI AGI of $29,981.71 to $16,360.34 taxable, giving $584.74 of tax and $284.74 after the credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It subtracted Social Security and a standard deduction but left out the $700 + $250 exemptions and the $300 school property tax credit. $860 matches about $22,600 of taxable income at 3.5%/4.4% with no credit, against the reference's $16,360.34 taxable and $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It applied the $300 school property tax credit correctly, but its $25,235 of taxable income implies a sliding standard deduction of only ~$8,500 and no $950 of personal and age exemptions. At the reference's $29,981.71 WI AGI the single sliding deduction is $12,671.36, which leaves $16,360.34 taxable and $584.74 of tax."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,973 matches Wisconsin taxable income in the $40,000-plus range, which only happens if taxable Social Security stays in the base and the standard deduction, exemptions, and $300 credit are skipped. Wisconsin subtracts the $13,655.95 of taxable Social Security, leaving WI AGI of $29,981.71 and $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It used the right structure (subtract taxable Social Security, standard deduction, senior exemption) on WI AGI of $33,735 but never applied the $300 school property tax credit, so its $786 is a pre-credit tax. The reference applies the $300 credit to $584.74 of tax on $16,360.34 taxable (WI AGI $29,981.71)."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation states a net tax of $1,617 but it submitted $1,348. After adding back a $300 credit, both figures require Wisconsin taxable income above $40,000, which means taxable Social Security stayed in the base. The reference subtracts the $13,655.95 of taxable Social Security and reaches $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,428 after the school property tax credit implies a ~$1,728 tentative tax, which requires Wisconsin taxable income of about $42,000 — only reachable if the $13,655.95 of taxable Social Security is left in the base. The reference subtracts it, giving WI AGI of $29,981.71, $16,360.34 taxable, and $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It subtracted essentially all IRA and pension income as a Wisconsin senior retirement subtraction, which cut WI AGI to ~$9,720 and the tax to $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, leaving WI AGI of $29,981.71 and $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It gave a placeholder-like $123.45 with no computation, assuming a retirement subtraction and credits leave only a small amount. The reference subtracts only taxable Social Security, leaving $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 school property tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It reached $0 by applying pension exclusions on top of the standard deduction. The reference applies no retirement subtraction, only the $13,655.95 taxable Social Security subtraction, leaving $16,360.34 taxable and $284.74 after the $300 school property tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It claimed without any computation that deductions and credits offset all of the tax. The $12,671.36 standard deduction and $950 of exemptions leave $16,360.34 taxable on WI AGI of $29,981.71, and the $300 school property tax credit covers only part of the resulting $584.74."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It treated the school property tax credit on $5,011 of real estate taxes as large enough to wipe out the tentative tax. The credit is 12% of the first $2,500 of property taxes, capped at $300, while tax on $16,360.34 taxable is $584.74, so $284.74 remains."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It said no Wisconsin nonrefundable credit applies, missing the $300 school property tax credit on $5,011 of real estate taxes. Its ~$800 is a pre-credit tax on WI income that includes the $3,753 of capital gains, above the reference's $584.74 on $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It applied Wisconsin's age-67+ retirement-income subtraction of up to $24,000 to the $24,086 of IRA and pension income, so the remaining income fell within the standard deduction and tax was $0. The reference's only subtraction is the $13,655.95 of taxable Social Security, so WI AGI stays at $29,981.71 and the result is $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"$430 matches the reference's method (subtract taxable Social Security, sliding standard deduction, $950 exemptions, $300 school property tax credit) applied to WI income that includes the $3,753 of capital gains less the 30% long-term exclusion (~$32,609 WI AGI, ~$19,300 taxable). The reference's federal AGI of $43,637.65 contains no capital gains, so its WI AGI is $29,981.71 and its taxable income is $16,360.34."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67 retirement-income subtraction plus a capital-gain subtraction, which left income below the standard deduction and tax at $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, giving $16,360.34 taxable and $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It excluded Social Security and applied the standard deduction and a capital-gain adjustment, but included no nonrefundable credit, so $770 is a pre-credit tax. The $300 school property tax credit applies; the reference's tax before that credit is $584.74 on $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It reached $0 by subtracting the IRA and pension income under an age-based Wisconsin retirement-income subtraction before the standard deduction and exemptions. The reference subtracts only the $13,655.95 of taxable Social Security, so WI AGI is $29,981.71, taxable income is $16,360.34, and the result is $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It combined a retirement-income deduction with the standard deduction, exemptions, and credits to reach $0. The reference applies no retirement subtraction; WI AGI of $29,981.71 less $12,671.36 and $950 leaves $16,360.34 taxable, and the $300 school property tax credit covers only part of the $584.74 tax."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It relied on Wisconsin retirement exclusions plus the standard deduction to bring the tax to $0. The reference's only subtraction is the $13,655.95 of taxable Social Security, giving WI AGI of $29,981.71, $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It used a $366 school property tax credit, which exceeds the $300 cap (12% of the first $2,500 of property tax), and a $10,683 standard deduction, below the single sliding amount (~$12,356 at its own $32,680 WI AGI). Its WI AGI also includes the $3,753 of capital gains, which the reference's $29,981.71 does not."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It followed the reference's steps (subtract taxable Social Security, single sliding standard deduction, $950 exemptions, 3.5%/4.4% rates, $300 school property tax credit) on WI AGI of $32,680, which includes the $3,753 of capital gains less the 30% long-term exclusion. The reference's federal AGI of $43,637.65 contains no capital gains, giving WI AGI of $29,981.71 and $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"Its method matches the reference's (taxable Social Security subtraction, $12,322 phased single standard deduction, $950 exemptions, 3.5%/4.4% rates, $300 school property tax credit), but it works from WI AGI of $32,680, which includes the $3,753 of capital gains less a $1,126 exclusion. The reference's WI AGI of $29,981.71 carries no capital gains, so its taxable income is $16,360.34 rather than $19,408."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It used the pre-2023 Wisconsin rates of 3.54%/4.65%, an unphased $13,900 standard deduction, no $950 of personal and age exemptions, and no $300 school property tax credit. The reference phases the deduction to $12,671.36, subtracts $950, taxes $16,360.34 at 3.5%/4.4%, and subtracts the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed the second bracket at 4.65% instead of 4.4% and used an ~$11,351 standard deduction, below the single sliding-scale amount at its income. Its WI income of ~$32,679 also includes the $3,753 of capital gains, which the reference's $29,981.71 WI AGI does not."
+us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It followed the reference's steps (subtract taxable Social Security, single sliding standard deduction phased at 12%, $950 exemptions, 3.5%/4.4% brackets, $300 school property tax credit) on WI income of $32,609, which includes the $3,753 of capital gains less the 30% long-term exclusion. The reference's federal AGI of $43,637.65 contains no capital gains, so its WI AGI is $29,981.71 and its tax is $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It applied a $24,000 retirement-income exclusion to the IRA and pension income, which dropped Wisconsin income to ~$9,806, then used an inflated $24,000 surviving-spouse standard deduction to reach $0. The reference subtracts only the $13,655.95 of taxable Social Security and uses the $12,671.36 single sliding deduction, leaving $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It used a married-joint standard deduction (~$13,300), taxed all $20,506 at 3.5% while ignoring the 4.4% bracket above ~$15,000, and invented an income phase-out that cut the school property tax credit to ~$276. The credit is a flat 12% of the first $2,500 of property taxes ($300), and the reference's single sliding deduction leaves $16,360.34 taxable."
+us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Its arithmetic contradicts itself: it computes $297 of tax and ~$339 of credits but reports $417.88. It also uses a $10,430 head-of-household deduction and invents a 10.82% personal tax credit and a $307 itemized deduction credit on real estate taxes, which do not qualify. The reference uses the $12,671.36 single deduction and only the $300 school property tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,830 matches about $44,700 of Wisconsin taxable income at 3.5%/4.4% with no credit, which requires leaving taxable Social Security in the base and skipping the standard deduction and exemptions. The reference subtracts the $13,655.95 of taxable Social Security, the $12,671.36 deduction, and $950 of exemptions, leaving $16,360.34 taxable."
us,scenario_042,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model understated Wisconsin homestead-credit household income by excluding substantial Social Security, taxable IRA distributions, pension income, dividends, interest, and capital gains, then applied an assumed 2024 maximum-credit formula instead of the 2026 eligibility computation. The complete computation produces no Wisconsin refundable credit."
us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric state_refundable_credits output or explanation, violating the required structured-output contract."
us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model applied the childless EITC phase-in (7.65% × $4,073) without checking IRC §32(c)(1)(A)(ii)(II), which limits the no-qualifying-children EITC to filers at least 25 and under 65 at the close of the tax year; the head is 66, so the credit is $0 for 2026. Its reasoning tested only the earned-income and AGI phase-out thresholds and never the upper age bar, treating the ARPA-2021 suspension of that bar as if it still applied in 2026. With EITC out, all five refundable components (EITC, refundable AOTC, refundable CTC, Recovery Rebate, refundable payroll tax credit) are $0."
@@ -3721,119 +3827,82 @@ us,scenario_043,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibi
us,scenario_043,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_043,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_043,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It counted FICA ($312.73) plus a Colorado FAMLI employee premium at 0.45% ($18.40) and reached $331.14 in its reasoning, within $1 of the reference ($330.73, which counts the most the employer may deduct, 0.44% of wages). It then submitted $315.14, a number its own calculation does not support, so the miss comes from that figure, not the rate."
-us,scenario_043,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It assumed no modeled mandatory Colorado employee payroll tax and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It said no mandatory state payroll taxes apply in Colorado and submitted FICA only, $312.74 rounded to $313. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It computed a Colorado FAMLI employee premium (0.45% × $4,088 = $18.40), then left it out as like state unemployment insurance and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It said Colorado has no mandatory employee payroll tax applicable here and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It computed only employee FICA on $4,088, about $313, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It treated employee payroll tax as 7.65% FICA only, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It said Colorado has no mandatory employee-side state payroll tax and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said Colorado has no mandatory employee-side state payroll tax, no state disability or unemployment-insurance employee contribution, and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said Colorado has no employee state payroll tax and submitted 7.65% FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It included a Colorado FAMLI employee premium but priced it at $24.53, which is 0.6% of the $4,088 in wages, for $337.26. The reference counts the most the employer may deduct, half the 0.88% premium, or 0.44% of wages ($17.99), so the model's FAMLI is $6.54 above it."
-us,scenario_043,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It subtracted the full $8,389 employer-sponsored insurance premium from wages, cut the FICA and FAMLI wage base to $0 and returned $0. The engine applies Social Security and Medicare to the full $4,088.15 of wages, $312.74, and the reference adds $17.99 of FAMLI (0.44% of wages, the most the employer may deduct, which the law lets the employer pay itself), for $330.73."
-us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA to $4,088 of wages, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It summed only employee Social Security and Medicare tax, $312.74, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It computed employee FICA only, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to wages, $312.73, rounded to $313, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It said no state employee payroll tax applies in Colorado and submitted FICA only, truncated to $312. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It returned $0, treating the wages as not subject to employee payroll taxes, without computing anything. The engine applies 6.2% Social Security and 1.45% Medicare to the full $4,088.15 of wages, $312.74, and the reference adds $17.99 of FAMLI (0.44% of wages, the most the employer may deduct, which the law lets the employer pay itself), for $330.73."
-us,scenario_043,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA, $312.73, and said it included no mandatory Colorado employee payroll tax. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It computed only 7.65% FICA on $4,088, $312.732, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It computed only Social Security and Medicare, $312.73, rounded to $313, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It kept the 401(k) deferrals in the FICA base and said no Colorado employee payroll tax applies, submitting FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It said no Colorado employee payroll tax applies and submitted 7.65% FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It said no mandatory state payroll taxes apply and rounded FICA of $312.732 to $313. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It computed employee FICA only, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It said Colorado has no mandatory employee state payroll tax and rounded FICA to $313. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It included only Social Security and Medicare, $312.73, and did not mention FAMLI. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It said Colorado has no mandatory employee state payroll tax and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
-us,scenario_043,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It said Colorado does not impose a mandatory employee payroll income tax and submitted FICA only, $312.73. The reference adds $17.99 (0.44% of wages), the most the employer may deduct for Colorado FAMLI, half the 0.88% premium. The law lets the employer pay that share itself, and the prompt does not say which it does; counting only what the law requires of the employee gives federal tax alone: $312.74 in the engine, $312.73 on the prompt's rounded $4,088 of wages."
+us,scenario_043,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model included FAMLI, but it used the old 0.45% employee rate instead of the 2026 rate of 0.44% (half of 0.88%), which gave $18.40 instead of $17.99. It then reported $315.14, which matches neither its own $331.14 total nor its $312.73 FICA-only figure. That is a transcription and arithmetic error on top of the rate error."
+us,scenario_043,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"FICA of $312.73 is computed correctly. The model left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference counts as a mandatory employee state payroll tax. The model instead took the reading that the employer absorbs that share."
+us,scenario_043,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee state payroll tax, so it left out the FAMLI employee premium of 0.44% × $4,088.15 = $17.99. It also rounded FICA to $313. The reference counts the FAMLI employee share in payroll tax."
+us,scenario_043,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model counted only federal FICA (about $312.74, rounded to $313). It left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference treats as a mandatory employee state payroll tax."
+us,scenario_043,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model recognized FAMLI's 50/50 split but then dropped it as 'state UI-like'. It also used 0.45% instead of the 2026 rate of 0.44%. The reference keeps the $17.99 FAMLI employee premium (0.44% × $4,088.15) as a mandatory employee state payroll tax, so the model's $312.73 leaves it out."
+us,scenario_043,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee payroll tax, which overlooks the FAMLI employee premium of 0.44% × $4,088.15 = $17.99. The reference adds that amount to the $312.74 of FICA."
+us,scenario_043,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model computed only FICA (about $313) and never added Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99). The reference counts that premium as a mandatory employee state payroll tax."
+us,scenario_043,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model correctly applied 7.65% FICA to the full $4,088, without reducing it for 401k deferrals. It left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference includes as an employee state payroll tax."
+us,scenario_043,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee-side payroll tax, so it left out the FAMLI employee premium (0.44% × $4,088.15 = $17.99) that the reference adds to $312.74 of FICA."
+us,scenario_043,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model looked only for state disability or unemployment-insurance employee contributions and missed Colorado's FAMLI paid-leave premium. The employee share of that premium is 0.44% × $4,088.15 = $17.99, and the reference includes it."
+us,scenario_043,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no employee state payroll tax, so it reported FICA only ($312.73). The reference adds the FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model correctly included Colorado's FAMLI employee premium but charged $24.53, which is 0.6% of wages. The 2026 employee rate is 0.44% (half of the 0.88% premium), which gives $17.99, so the model overstated the total by $6.53."
+us,scenario_043,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model treated the $8,389 of employer-sponsored insurance premiums as a pre-tax Section 125 deduction that wipes out FICA wages, which it got to $0. Nothing in the household facts makes those premiums a salary reduction. The full $4,088 of wages is subject to FICA ($312.74) and to the FAMLI employee premium ($17.99)."
+us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate to $4,088. It left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference counts as an employee state payroll tax."
+us,scenario_043,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model's FICA of $312.74 matches the reference's federal components. It left out the $17.99 Colorado FAMLI employee premium (0.44% of $4,088.15 in FAMLI wages)."
+us,scenario_043,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model reported FICA only ($312.73) and did not add Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99). The reference includes that premium as a mandatory employee state payroll tax."
+us,scenario_043,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model computed Social Security and Medicare correctly but never addressed state payroll tax. It therefore left out the Colorado FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model applied 7.65% FICA and rounded to $313. It left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference adds."
+us,scenario_043,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no state employee payroll tax and left out the FAMLI employee premium (0.44% × $4,088.15 = $17.99). It also truncated FICA of $312.73 down to $312."
+us,scenario_043,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model gave no basis for treating $4,088 of wages as exempt from payroll tax and reported $0. Every dollar of those wages is subject to 6.2% Social Security, 1.45% Medicare, and the 0.44% Colorado FAMLI employee premium."
+us,scenario_043,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model chose to leave out any Colorado employee payroll tax and reported 7.65% × $4,088 = $312.73. The reference includes the FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model computed FICA only ($312.73) and left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference counts as a mandatory employee state payroll tax."
+us,scenario_043,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model applied 7.65% FICA and rounded to $313. It did not add Colorado's FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model handled the FICA base correctly but said Colorado has no employee payroll tax. That omits the FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference includes."
+us,scenario_043,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model correctly kept 401k deferrals in FICA wages but said there is no Colorado employee payroll tax. It therefore left out the FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model said no mandatory state payroll tax applies and rounded FICA to $313. That omits Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99)."
+us,scenario_043,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model computed FICA only ($312.73) and never considered Colorado's FAMLI paid-leave premium. The employee share is 0.44% × $4,088.15 = $17.99, and the reference includes it."
+us,scenario_043,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee state payroll tax and rounded FICA to $313. It left out the FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
+us,scenario_043,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model computed Social Security and Medicare correctly but never added Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference counts in payroll tax."
+us,scenario_043,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee state payroll tax and reported FICA only. That omits the FAMLI employee premium of 0.44% × $4,088.15 = $17.99, which the reference adds."
+us,scenario_043,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model ruled out a Colorado employee payroll tax and reported $312.73. The reference adds the FAMLI paid-leave employee premium of 0.44% × $4,088.15 = $17.99."
us,scenario_043,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the earned-income, standard and elderly excess medical deductions. It then used $291 a month, the FY2024 one-person maximum, instead of the 2026 maximum of $298. That gives $3,492 instead of $3,576."
-us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied the $2,250 general-household resource limit and concluded that $2,800 in bank assets makes the household ineligible. Because the 66-year-old head is elderly, the higher elderly resource limit applies, and $2,800 passes it. The household is also categorically eligible through Colorado's TANF non-cash test. It is eligible for the $298 monthly maximum."
-us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly found net income of $0 and a maximum of about $300 a month, which gives about $3,600. It then dropped that result and put in an unsupported $200 a month. With net income at $0, the benefit is the full one-person maximum of $298 a month."
-us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It found net income near zero and cited a maximum of about $292. It then invented a reduced benefit of $192 a month on the false idea that a minimum-benefit rule applies. When net income is $0, the household gets the full one-person maximum allotment, which is $298 in 2026."
-us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly found net income near zero after the elderly medical deduction. It then used a one-person maximum allotment of about $203 a month, far below the 2026 contiguous-states maximum of $298. That gives $2,436 instead of $3,576."
-us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly found that the household passes the asset test and has net income of $0. It then used $292 a month, the FY2025 one-person maximum, instead of the 2026 value of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly found net income of $0 and eligibility for the maximum allotment. It then overstated the one-person maximum as about $328 a month instead of $298, giving $3,936."
-us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly applied the elderly asset limit and the deductions and found net income of $0. It then used $301 a month as the 2026 one-person maximum instead of $298, giving $3,612."
-us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It found net income below zero but said outright that it was using the FY2025 maximum of $292. The 2026 one-person maximum of $298 applies, so it answered $3,504 instead of $3,576."
-us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly treated this one-person elderly household as receiving the maximum benefit. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298."
-us,scenario_043,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly found net income below zero and awarded the maximum allotment. It then put the 2026 one-person maximum at $299 a month instead of $298, giving $3,588 instead of $3,576."
-us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly brought net income to $0 through the elderly medical, standard and earned-income deductions. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504."
-us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It claimed the household's assets and income disqualify it. The household's gross income of $387 a month is about 30% of the poverty guideline, and its $2,800 in assets is under the elderly resource limit. It is also categorically eligible through Colorado's TANF non-cash test, so it gets the $298 monthly maximum."
-us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the elderly medical, standard and earned-income deductions. It then used $291 a month, the FY2024 one-person maximum, instead of the 2026 value of $298, giving $3,492."
-us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly found categorical eligibility under Colorado's rules and net income of $0. It then gave a rounded $3,500, which is about $292 a month, instead of the actual 2026 maximum of $298 × 12 = $3,576."
-us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It answered $0 on vague income and resource grounds. The household passes the gross income, net income and asset tests and is categorically eligible through Colorado's TANF non-cash test. With net income at $0, it gets the $298 monthly maximum."
-us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly computed a $381.67-a-month excess medical deduction that brings net income to $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504."
-us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the standard, earned-income and excess medical deductions. Its $3,504 answer is $292 a month, the FY2025 maximum, instead of the 2026 one-person maximum of $298."
-us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly found that medical expenses bring net income to $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504."
-us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"It correctly found net income of $0 and awarded the maximum allotment. Its $3,624 implies $302 a month, which overstates the 2026 one-person maximum of $298."
-us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,"It correctly brought net income to $0 through the elderly medical deduction. It then overstated the one-person maximum as about $305 a month instead of $298, giving $3,660."
-us,scenario_043,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave an unsupported $1,276, about $106 a month, without computing net income. It never applied the elderly excess medical deduction on $5,000 of out-of-pocket medical costs or the standard and earned-income deductions. Those bring net income to $0, which yields the full $298 monthly maximum."
-us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It said the household is ineligible because it lacks low-income indicators. Its gross income of $387 a month is about 30% of the poverty guideline, which is far below every income limit. The household is also categorically eligible through Colorado's TANF non-cash test and receives the $298 monthly maximum."
-us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"It claimed the $2,800 in assets exceeds typical limits. Because the head is 66, the household is elderly and $2,800 is under the elderly resource limit. The household is also categorically eligible through Colorado's TANF non-cash test. It also ignored the $5,000 in stated medical expenses, which feed the excess medical deduction that brings net income to $0."
-us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly found that the household passes all tests and has net income of $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 one-person maximum of $298, giving $3,504."
-us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly applied the elderly resource limit and the three deductions and found net income of $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298."
-us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly found net income of $0 and eligibility for the maximum. It then approximated the 2026 one-person maximum as $300 a month instead of $298, giving $3,600 instead of $3,576."
-us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the earned-income, standard and elderly medical deductions. It then rounded the one-person maximum to about $300 a month instead of the actual $298, giving $3,600."
-us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there is nothing to score. The correct derivation is net income of $0 for this elderly one-person household, giving the $298 monthly maximum, or $3,576 a year."
-us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"It listed the income and assets but asserted $0 without applying any test. Gross income is about 30% of the poverty guideline, and $2,800 is under the elderly resource limit. After the deductions, net income is $0, so the household receives the $298 monthly maximum."
-us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It filed the medical expenses under the shelter deduction instead of the separate elderly excess medical deduction. It then used an understated 2026 one-person maximum of about $243 a month. The correct maximum is $298, and it applies in full because net income is $0."
-us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It said without reasoning that PolicyEngine produces no benefit. The household passes the income and asset tests, is categorically eligible through Colorado's TANF non-cash test, and has net income of $0 after the elderly medical deduction. It therefore receives $298 a month."
+us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to $0 using the earned income, standard and excess medical deductions. It then used a stale $291/month one-person maximum allotment instead of the $298 FY2026 figure, which gave $3,492 instead of $3,576."
+us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model gave $0 by claiming a $2,250 asset limit that $2,800 exceeds. For a household with a member aged 60 or older, the limit is $4,500, and Colorado's TANF-based categorical eligibility removes the asset test anyway. The household is eligible and gets the $298/month maximum."
+us,scenario_043,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model gave $0 by citing elderly asset/income tests and an absence of take-up. The prompt explicitly assumes take-up, and the household passes every test: it is categorically eligible through Colorado's broad-based categorical eligibility, and $2,800 is under the elderly asset limit. With net income at $0, it gets the full $298/month."
+us,scenario_043,snap,claude-opus-4.7,llm_error,other,False,"The model correctly found net income of $0 and a maximum benefit near $300/month ($3,600/yr). It then arbitrarily changed its estimate to $200/month. With net income of $0, the benefit is the full $298 maximum allotment, not a reduced amount."
+us,scenario_043,snap,claude-opus-4.8,llm_error,other,False,"The model found very low net income and cited a $292 maximum. It then gave an unsupported $192/month, apparently confusing the benefit with a minimum-benefit rule. With net income of $0, the benefit equals the full maximum allotment, which is $298 in FY2026."
+us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly found net income near $0 after the excess medical deduction. It then used a maximum allotment of about $203/month, far below the actual $298 one-person maximum, which understated the annual benefit by more than $1,100."
+us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly reduced net income to $0 and identified the maximum-allotment outcome. It used the FY2025 one-person maximum of $292/month instead of the FY2026 $298, giving $3,504 instead of $3,576."
+us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly found net income of $0 but overstated the one-person maximum allotment as about $328/month. The actual FY2026 contiguous-states maximum is $298/month ($3,576/yr)."
+us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model got the eligibility tests and net income of $0 right but used $301/month as the 2026 one-person maximum. The actual figure is $298/month, so its answer was $36 too high for the year."
+us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model explicitly applied the FY2025 maximum allotment of $292/month to a 2026 benefit year. The FY2026 one-person maximum is $298/month, which gives $3,576."
+us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly concluded the household gets the maximum benefit but used the FY2025 $292/month figure. The FY2026 one-person maximum is $298/month ($3,576/yr)."
+us,scenario_043,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model correctly found net income below zero but recalled the FY2026 one-person maximum as $299/month. The actual figure is $298/month, so the annual total was $12 too high."
+us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model reduced net income to $0 through the elderly medical deduction but used the FY2025 maximum of $292/month. The FY2026 one-person maximum is $298/month ($3,576/yr)."
+us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model claimed the household's assets and income disqualify it. Monthly income of $387 is about 30% of the poverty guideline, $2,800 is under the elderly asset limit, and Colorado's categorical eligibility applies. The household is eligible for the full $298/month maximum."
+us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly found net income of $0 but used the outdated $291/month one-person maximum. The FY2026 figure is $298/month, which gives $3,576 instead of $3,492."
+us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified categorical eligibility and net income of $0 but submitted a rounded $3,500, equal to about $291.67/month. The answer should have been twelve months of the $298 FY2026 maximum allotment ($3,576)."
+us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that income and resource rules give $0. The household's $387/month income is far below the limits, its $2,800 in assets is under the elderly limit, and it is categorically eligible in Colorado. With net income of $0, it gets the $298/month maximum."
+us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly computed gross income and the $381.67 excess medical deduction, reaching net income of $0. It then applied the FY2025 $292/month maximum instead of the FY2026 $298."
+us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly reached net income of $0 but its $3,504 total reflects the FY2025 $292/month maximum. The FY2026 one-person maximum is $298/month."
+us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly found net income of $0 but used the $292/month FY2025 maximum allotment. The FY2026 one-person maximum is $298/month ($3,576/yr)."
+us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly concluded net income is $0 and the maximum allotment applies. Its $3,624 implies $302/month, above the actual FY2026 one-person maximum of $298/month."
+us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,The model correctly drove net income to $0 with the elderly medical deduction but overstated the one-person maximum allotment as $305/month. The actual figure is $298.
+us,scenario_043,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model's $1,276 implies about $106/month, as if a large share of income were counted against the benefit. It skipped the elderly excess medical deduction ($381.67/month), which together with the standard and earned income deductions brings net income to $0 and gives the full $298/month."
+us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed there were no low-income indicators. The household's income is $387/month, about 30% of the poverty guideline, and it is categorically eligible through Colorado's TANF non-cash pathway, so it gets the $298/month maximum."
+us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model claimed $2,800 in assets exceeds the limits. It is under both the $3,000 general limit and the $4,500 limit for a household with an elderly member, and Colorado's broad-based categorical eligibility removes the asset test. The medical expenses are listed facts, not unlisted ones."
+us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,The model correctly applied the eligibility tests and the excess medical deduction to reach net income of $0. It then used the FY2025 $292/month maximum instead of the FY2026 $298/month.
+us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"The model got the resource test and deductions right, reaching net income of $0. It then used the FY2025 $292/month one-person maximum rather than the FY2026 $298/month."
+us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly found net income of $0 but rounded the one-person maximum allotment to about $300/month. The actual figure is $298/month, which gives $3,576 rather than $3,600."
+us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"The model correctly reached net income of $0 via the elderly medical deduction but approximated the maximum allotment as $300/month. The FY2026 one-person contiguous-states maximum is $298/month ($3,576/yr)."
+us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value or explanation, so there was no answer to score. The correct derivation gives twelve months of the $298 one-person maximum, or $3,576."
+us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly noted that elderly households face more lenient limits but then asserted a $0 benefit without any calculation. Income of $387/month and $2,800 in assets pass every test, and net income of $0 gives the $298/month maximum."
+us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model treated medical premiums as a shelter item and used a maximum allotment of about $243/month. The $381.67 excess medical deduction brings net income to $0, so the benefit is the actual $298/month maximum."
+us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted without derivation that no positive benefit results. The household is categorically eligible in Colorado, passes the elderly asset limit, and has net income of $0 after deductions, so it gets $298/month ($3,576/yr)."
us,scenario_043,ssi,claude-fable-5.1,llm_error,period_annualization,False,"The model annualized the monthly SSI federal benefit rate and monthly income exclusions, then mechanically subtracted annual countable income to produce $9,954. It also identified that $2,800 exceeds the stated $2,000 resource limit but discarded that result; the controlling engine computation finds the individual’s approximately $4,642 of income above the SSI income limit, yielding zero."
us,scenario_043,ssi,claude-opus-4.7,llm_error,period_annualization,False,"The model converted annual wages and pension income into a monthly countable-income figure and subtracted it from an estimated monthly federal benefit rate, bypassing the SSI income-eligibility gate. The individual’s approximately $4,642 of income exceeds the applicable limit, so aged status and assumed take-up do not produce a payable benefit."
us,scenario_043,ssi,claude-opus-5,llm_error,period_annualization,False,"The model first annualized the SSI benefit rate and exclusions to derive about $9,360, then replaced that calculation with an unsupported $6,720 figure. Both paths miss that the individual’s approximately $4,642 of income fails the SSI income-limit test, which eliminates the benefit entirely."
us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, so it failed the required structured-output contract."
-us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Correctly zeroed the Colorado EITC because a 66-year-old is past the federal childless EITC age cap. It then left out the Colorado sales tax refund, a refundable credit that pays $19 at modified AGI of $4,625 in 2026, and wrongly said the income was too low to trigger any other credit."
-us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Invented a Colorado EITC for childless filers aged 65+. Colorado's EITC is a percentage of the federal EITC, which is $0 for this 66-year-old, so its $78.18 has no basis. It also missed the only credit that applies: the $19 sales tax refund."
-us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly claimed Colorado has no refundable individual credits. The state EITC, child tax credit, family affordability credit and sales tax refund are all refundable, and the sales tax refund pays this filer $19."
-us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Picked the right program, the TABOR sales tax refund, but invented a lowest-tier amount of about $1,130. For 2026, the tier for modified AGI of $4,625.40 pays $19."
-us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Mentioned TABOR refunds but wrongly assumed they depend on earned income or other triggers. Every full-year resident filer gets the sales tax refund, and at modified AGI of $4,625.40 it pays $19."
-us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Made up a $225 refundable senior housing or income-based credit that pays nothing in 2026. It missed the only credit that applies: the $19 Colorado sales tax refund at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Named the 2026 TABOR sales tax refund but chose to leave it out. This filer qualifies for it, and at modified AGI of $4,625.40 it pays $19."
-us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly zeroed the Colorado EITC and child tax credit. It then said no other refundable credit applies and missed the sales tax refund, which pays this 66-year-old resident filer $19."
-us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Invented a flat refundable senior credit of about $1,200. The senior housing credit pays nothing in 2026, and the TABOR sales tax refund at modified AGI of $4,625.40 is only $19."
-us,scenario_043,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Said no Colorado refundable credit applies and missed the sales tax refund, which pays this resident filer $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Correctly ruled out the Colorado EITC because the head is 66, but wrongly said no other refundable credit exists. The Colorado sales tax refund pays this filer $19."
-us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Checked only the EITC and child-based credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Assumed Colorado's refundable credits need qualifying children. The sales tax refund goes to childless resident filers and pays this household $19.
-us,scenario_043,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Stopped once the Colorado EITC came out at $0 and missed the refundable sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,Looked only at the Colorado EITC and child-related credits and missed the Colorado sales tax refund. That refund does not depend on EITC eligibility and pays $19 here.
-us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Said income and household makeup rule out every Colorado refundable credit. In fact the sales tax refund goes to low-income single resident filers and pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Checked only the property tax, rent and heat rebate and ignored the Colorado sales tax refund. That refund needs no qualifying expenses and pays $19 here."
-us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,Wrongly tied all Colorado refundable credits to federal EITC eligibility. The sales tax refund does not depend on the EITC and pays this filer $19.
-us,scenario_043,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Answered $0 without identifying any credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,Answered $0 without naming a program and missed the Colorado sales tax refund. That refund goes to every full-year resident filer and pays $19 at this income.
-us,scenario_043,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Based the answer only on the missing children and the $0 federal EITC, and missed the Colorado sales tax refund. That refund pays $19 regardless of those factors."
-us,scenario_043,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,Ruled out the EITC and state child tax credit but never included the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19.
-us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly applied the age limit to zero out the EITC, then wrongly said no other refundable credit applies. The Colorado sales tax refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"Assumed a $625 federal EITC even though a 66-year-old is past the childless EITC age limit, so federal and Colorado EITC are both $0. It also missed the only credit that applies: the $19 sales tax refund."
-us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Said none of the reported facts trigger a refundable Colorado credit and missed the sales tax refund. Filing as a full-year adult resident is enough, and it pays $19 here."
-us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed refundable credits need earned-income eligibility or children. It missed the Colorado sales tax refund, which pays this filer $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Checked the child, earned-income and property/rent/heat credits but left out the Colorado sales tax refund. That refund needs no expense facts and pays $19 here."
-us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Said no Colorado refundable credit applies and missed the sales tax refund, which pays this resident filer $19."
-us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Looked only for EITC, dependent-based and expense-based credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Said none of the listed facts support a refundable credit. Colorado residency plus filing is enough for the sales tax refund, which pays $19 here."
-us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Gave the maximum $800 senior housing income tax credit, but that credit pays nothing for tax year 2026. The only Colorado refundable credit here is the $19 sales tax refund."
-us,scenario_043,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Correctly ruled out the EITC because of the head's age and the credits because of no children, but missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,Looked only for dependent-based and housing-cost-based credits and missed the Colorado sales tax refund. That refund needs neither and pays $19 here.
-us,scenario_043,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Covered the child, EITC and property/rent/heat credits but left out the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19."
-us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Answered $0 and missed the Colorado sales tax refund, a refundable state credit that pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,Ruled out the EITC match and child credits but never included the Colorado sales tax refund. That refund goes to every full-year resident filer and pays $19 here.
-us,scenario_043,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Correctly zeroed the EITC and the senior property tax, rent and heat rebate but missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Checked the EITC, the child tax credit and the property tax, rent and heat rebate but left out the Colorado sales tax refund. That refund pays this filer $19."
-us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"Assumed a $313 federal EITC for a 66-year-old childless filer who is past the EITC age limit. Its 50% Colorado match therefore has no basis, since the Colorado EITC is $0. It also missed the $19 sales tax refund."
-us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,Correctly zeroed the Colorado EITC and child tax credit but missed the Colorado sales tax refund. That refund does not depend on age limits or dependents and pays $19.
-us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,Assumed a refundable credit needs dependents or qualifying expenses. The Colorado sales tax refund needs neither and pays this resident filer $19.
-us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Tied Colorado's refundable credits only to the federal EITC, CTC, children and childcare costs, and missed the sales tax refund. That refund pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Said the low income and missing children rule out every Colorado credit. In fact the sales tax refund is aimed at low-income filers and pays $19 here.
-us,scenario_043,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Considered the TABOR sales tax refund and wrongly declared it inapplicable for 2026. This full-year resident filer qualifies, and it pays $19 at modified AGI of $4,625.40."
-us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Checked only the child tax credit and the EITC and missed the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19.
-us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Wrongly tied refundable credits to Colorado tax liability. Refundable credits pay out even when liability is zero, and the sales tax refund pays this filer $19."
+us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,age_disability,False,"The model assumed Colorado's EITC covers childless workers aged 65 and older who are shut out federally only by age. Under PolicyEngine's Colorado rules, the state EITC is a percentage of the federal EITC actually allowed. The head is 66, above the federal 25–64 childless age window, so the federal EITC is $0 and 25% of $0 is $0, not 25% of the $312.73 phase-in amount."
+us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model paid a $1,130 lowest-tier TABOR sales tax refund without checking the surplus trigger. C.R.S. 39-22-2003(2) pays that refund for a tax year only if the fiscal year ending in it has revenue above the Referendum C cap. FY2025-26 was forecast below the cap, so the tax year 2026 refund is $0."
+us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model made up a $225 'senior/family affordability' refundable amount without naming a credit or a formula. The Family Affordability Credit requires a child under 17, and the senior housing and sales tax refunds are TABOR-surplus mechanisms that pay nothing in 2026. That leaves this childless senior with $0 in Colorado refundable credits."
+us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the Senior Housing Income Tax Credit and the TABOR sales tax refund as guaranteed flat payments to low-income seniors and estimated $1,200. Both are paid only when Colorado has excess revenue under the Referendum C cap, and FY2025-26 had none, so neither pays for tax year 2026. The Colorado EITC is also $0 because a 66-year-old childless filer gets no federal EITC."
+us,scenario_043,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"The model took 25% of a $625 federal EITC without applying the federal childless-EITC age ceiling of 64. At 66 the head has no federal EITC, so the Colorado match is $0. The $625 figure also did not come from the head's $4,088 of earnings, since the phase-in amount at that income is about $313."
+us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model gave the head the maximum $800 Senior Housing Income Tax Credit just because the head is a low-income senior. That credit is funded through the TABOR surplus refund mechanism and pays nothing when the fiscal year ending in the tax year is under the Referendum C cap, which is the case for 2026. Colorado refundable credits are therefore $0."
+us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model computed a $313 federal childless EITC (7.65% of $4,088) and took 50% of it. It ignored that federal childless EITC eligibility ends at age 64. The 66-year-old head gets no federal EITC, so the Colorado EITC is $0, and the 50% match rate is also wrong for 2026."
us,scenario_043,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model took the full $56,806 of Social Security, subtracted the standard deduction, and taxed the remainder. It never applied the §86 provisional-income test: half of benefits, $28,403, is below the $32,000 joint base amount, so no benefits are includable and tax is $0."
us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied only the older-couple standard deduction and nonrefundable credits, and its $4,162 answer only works if a large share of the Social Security counts as taxable income. It skipped the benefit-taxation test. Provisional income of $28,403 is below the $32,000 MFJ base amount, so taxable income and tax are both $0."
@@ -3848,39 +3917,40 @@ us,scenario_044,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_erro
us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied the Kansas food sales tax credit ($125 per exemption, age 55+ pathway, AGI under $30,615) and got $250, but that credit ended after tax year 2024, when the state food sales tax went to 0%. It is not available in 2026. The household has no earned income for the Kansas EITC, so Kansas refundable credits are $0."
us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It gave the household the Kansas food sales tax credit of $125 for each of two exemptions based on the low-income, age 55+ test, but it used pre-2025 law. Kansas ended that credit after tax year 2024, when food became exempt from state sales tax, so it pays nothing in 2026. With no earnings there is no Kansas EITC either, so the correct total is $0."
us,scenario_044,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It correctly found Kansas AGI of $0 and both spouses over 55, but it then paid out the food sales tax credit (2 x $125 = $250), which Kansas ended after tax year 2024, when the state food sales tax rate went to zero. No refundable Kansas credit is left for this household in 2026. The Kansas EITC needs earned income, and the household has only Social Security."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It used the correct $16,100 standard deduction and the $12,400 10% bracket and derived $1,240 + $933 = $2,173. It then submitted $2,540 with no supporting computation, so the submitted value contradicts its own correct derivation."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the stated gross wages, which cut income to $15,068, and it applied the 2024 standard deduction of $14,600. It then wiped out the remaining tax with the EITC, a refundable credit that this pre-refundable-credit measure excludes. The correct base is the full $36,275.59 of wages less the $16,100 standard deduction, which gives $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"On its recheck it applied the $16,100 standard deduction and $12,400 bracket and got $2,173. It then submitted $1,631, a number unsupported by any step of its own derivation."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed taxable income of $20,176 and tax of $1,240 + $933 = $2,173 with no credits. It then submitted $1,387, citing 'rounding to bracket estimates', which no bracket rounding produces."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived $2,173 from the $16,100 standard deduction and the 10%/12% brackets. It then submitted $2,029 on an unexplained 'slight bracket rounding' that contradicts its own arithmetic."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the 2025 pre-OBBBA $15,000 standard deduction to $15,350 and used the 2025 10% bracket top of $11,925. The 2026 values are a $16,100 standard deduction and a $12,400 bracket top, so it overstated taxable income ($20,926 vs $20,175.59) and put too much in the 12% bracket."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used an understated $15,700 standard deduction and a $12,150 bracket top to reach $2,226, instead of $16,100 and $12,400. It then submitted $1,130, roughly half its own figure, with no computation behind it."
-us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the listed gross wages to get AGI of $15,068, then zeroed taxable income against the standard deduction. The listed gross wages are the AGI base ($36,275.59), and after the $16,100 standard deduction $20,175.59 is taxable."
-us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages to reach AGI of $15,068. It also applied a sunset-era $8,117 standard deduction and a $5,177 personal exemption. Under the permanent TCJA/OBBBA rules the 2026 base is $36,275.59, less a $16,100 standard deduction, with no personal exemption."
-us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,400 standard deduction, a $5,400 personal exemption and a 15% second bracket. The 2026 law is the permanent TCJA/OBBBA schedule: a $16,100 standard deduction, no exemption, and 10% up to $12,400 with 12% above."
-us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied a sunset-era $8,000 standard deduction and a $5,000 personal exemption with a 15% bracket above $11,650. The permanent TCJA/OBBBA 2026 rules use a $16,100 standard deduction, no exemption, and 12% above $12,400."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 employer-sponsored insurance premiums as a pre-tax reduction of the stated gross wages, which put AGI at $15,068, below its $15,700 standard deduction. AGI is the full $36,275.59 of wages, which leaves $20,175.59 taxable after the $16,100 deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It assigned head-of-household status to a single filer with no dependents and claimed that a $15,000 deduction eliminates tax on $36,276 of wages. Even that deduction leaves $21,276 taxable. The correct single-filer computation ($16,100 deduction, $20,175.59 taxable) yields $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: a standard deduction plus a personal exemption totaling about $14,050, and a 15% second bracket. The permanent TCJA/OBBBA 2026 schedule uses a $16,100 standard deduction, no exemption, and 10%/12% brackets split at $12,400."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from gross wages to reach AGI of $15,068, then used a sunset-era $13,350 standard deduction plus exemption. The correct base is $36,275.59 of wages less the $16,100 TCJA/OBBBA standard deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction exceeds taxable income for a single filer with $36,276 of wages. That is false: $36,275.59 less the $16,100 standard deduction leaves $20,175.59 taxable. A $0 result is reachable only by removing the $21,208 ESI premiums from wages."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced gross wages by the $21,208 ESI premiums to get AGI of $15,068, then applied a TCJA-expiration $8,300 standard deduction and a $5,150 personal exemption. The correct base is $36,275.59 of AGI less the permanent $16,100 standard deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used sunset-era parameters: an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, which put taxable income at $22,676. The permanent TCJA/OBBBA 2026 rules give $20,175.59 taxable, and tax of $2,173.07 at 10%/12%."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the 2025 pre-OBBBA $15,000 standard deduction and the 2025 $11,925 10% bracket top to tax year 2026. The 2026 values are $16,100 and $12,400, so it overstated taxable income by $1,100 and moved income into the 12% bracket."
-us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $15,400 standard deduction and a $12,250 10% bracket top for 2026 instead of the OBBBA-adjusted $16,100 and $12,400. That inflated taxable income to $20,876 and tax to $2,260.12."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed that the standard deduction and nonrefundable credits eliminate the tax. $36,275.59 of wages less the $16,100 standard deduction leaves $20,175.59 taxable, and a single childless filer has no nonrefundable credits to offset the resulting $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed without computation that the standard deduction and nonrefundable credits fully offset the tax. The $16,100 standard deduction leaves $20,175.59 taxable, and no nonrefundable credit applies to this childless single filer, so tax is $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 rather than the inflation-adjusted 2026 value of $16,100. Its $12,400 bracket top was correct, but the $350 shortfall in the deduction raised taxable income to $20,526 and tax to $2,215."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the stated gross wages to get $15,068, below the standard deduction. The listed gross wages are the AGI base of $36,275.59, leaving $20,175.59 taxable after the $16,100 deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized and treated the $21,208 employer-sponsored insurance premium as a deductible out-of-pocket medical expense above the 7.5%-of-AGI floor, claiming $18,487.30 of medical deductions. Excluding that premium, itemized deductions are only $1,541.71, so the $16,100 standard deduction applies, giving $20,175.59 taxable and $2,173.07 of tax."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA $15,000 standard deduction instead of the 2026 $16,100, which overstated taxable income at $21,276. It then produced $2,352, which exceeds even a correct 10%/12% bracket computation on that overstated base."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages to get AGI of $15,068 and assumed a TCJA sunset with an $8,535 standard deduction and a $5,444 exemption. The correct base is $36,275.59 of AGI less the permanent $16,100 standard deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the 2025 pre-OBBBA $15,000 standard deduction and the $11,925 10% bracket top to 2026 instead of $16,100 and $12,400. That produced $21,276 of taxable income and $2,315 of tax instead of $20,175.59 and $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,000 standard deduction, a $5,000 exemption and a 15% bracket above $11,600. The 2026 law keeps the TCJA structure permanently: a $16,100 standard deduction, no exemption, and 12% above $12,400."
-us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so no answer was parsed. The correct derivation is $36,275.59 of AGI less the $16,100 standard deduction, which leaves $20,175.59 taxed at 10%/12% for $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It correctly reached $20,176 of taxable income with the $16,100 standard deduction. It then zeroed the tax with the EITC, even though it acknowledged the EITC is refundable and excluded from this output. A childless 44-year-old earning $36,276 also receives no EITC."
-us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It used a stale $15,750 standard deduction and a $12,150 bracket top to compute $2,220.12. It then submitted $424.76, a value with no connection to its own computation."
-us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction exceeds AGI. $36,275.59 of wage AGI exceeds the $16,100 standard deduction by $20,175.59, so tax is $2,173.07. Its $0 is reachable only by removing the $21,208 ESI premiums from wages."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning gets the exact derivation: $16,100 standard deduction, $20,176 taxable income, and 1,240 + 0.12×7,776 = $2,173. It then submitted $2,540, a number its calculation does not support."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the gross wages, even though the given wages are the taxable wage base, and it used the outdated $14,600 standard deduction instead of the 2026 amount of $16,100. It then applied the refundable EITC to reduce the tax before refundable credits to zero. That EITC is both outside this output and unavailable to a childless filer earning $36k."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It correctly calculated taxable income of about $20,176 and tax of about $2,100 at the 10%/12% rates, and it noted that no nonrefundable credits apply. It still submitted 0, which contradicts its own calculation of about $2,173."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its corrected calculation was $1,240 + 0.12×($20,176−$12,400) = $2,173, which is the exact derivation. It then submitted $1,631, a figure that follows from no step it showed."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It calculated $1,240 + $933 = $2,173 using the correct $16,100 deduction and $12,400 10% bracket threshold, and it noted no credits apply. It then submitted $1,387, which its 'bracket rounding' does not explain."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It got about $2,173 from the correct $16,100 deduction and 10%/12% brackets. It then cited 'slight bracket rounding' to submit $2,029, which is about $144 below its own correct result."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction as $15,350 by inflating the 2025 amount, instead of using the OBBBA-set $16,100, and it used the 2025 10% bracket threshold of $11,925 instead of $12,400. Both errors overstate taxable income and push more of it into the 12% bracket, which gives $2,273."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used outdated parameters, a $15,700 deduction and a $12,150 10% bracket threshold, instead of $16,100 and $12,400, and got $2,226. It then submitted $1,130, which its own calculation does not support."
+us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 employer-sponsored insurance premiums as a pre-tax reduction to the stated gross wages, which gave an AGI of $15,068 and zero tax. AGI is the full $36,276 in wages, and after the $16,100 deduction, $20,176 is taxable."
+us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from wages, giving an AGI of $15,068. It also applied a pre-TCJA standard deduction and personal exemption, but OBBBA made the TCJA permanent, so 2026 has a $16,100 standard deduction and no exemption. The correct AGI is $36,276, and the tax is $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used an $8,400 standard deduction, a $5,400 personal exemption, and a 15% second bracket. OBBBA made the TCJA structure permanent, with a $16,100 standard deduction, no personal exemption, and 10%/12% brackets."
+us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules: an $8,000 standard deduction, a $5,000 exemption, and a 15% bracket above $11,650. Under OBBBA's permanent TCJA rules, the 2026 standard deduction is $16,100 and income above $12,400 is taxed at 12%."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 in employer-sponsored insurance premiums from the stated gross wages and got a $15,068 AGI below the standard deduction. AGI is the full $36,276, and after the $16,100 deduction, $20,175.59 is taxable."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It treated a filer with no dependents as head of household, and its own $15,000 deduction still leaves about $21,000 of taxable income, so its result of zero tax is false. The correct status is single: $36,276 − $16,100 = $20,176 taxable, and the tax is $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied 'pre-TCJA rules for 2026,' using a standard deduction plus a personal exemption and a 15% bracket. OBBBA made the TCJA permanent, so 2026 has a $16,100 standard deduction, no exemption, and a 12% second bracket."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from wages, giving a $15,068 AGI, and it applied a pre-TCJA deduction and exemption totaling $13,350. The full $36,276 in wages is AGI, and the OBBBA-permanent $16,100 standard deduction applies with no exemption."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction exceeds income, but the 2026 single deduction of $16,100 is far below $36,276 in wages. That leaves $20,175.59 of taxable income and $2,173.07 of tax."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $21,208 employer-sponsored insurance premiums to get a $15,068 AGI. It then applied a TCJA-expiration $8,300 deduction and $5,150 exemption, but OBBBA made the TCJA permanent, with a $16,100 deduction and no exemption, applied to the full $36,276 AGI."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used sunset-era parameters: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. OBBBA kept the TCJA structure, so 2026 taxable income is $36,276 − $16,100 at 10%/12%, giving $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 parameters, a $15,000 standard deduction and an $11,925 10% bracket threshold, instead of 2026's $16,100 and $12,400. That overstates taxable income by $1,100 and gives $2,314.62."
+us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It projected a $15,400 standard deduction and a $12,250 10% bracket threshold for 2026, but OBBBA raised the deduction to $16,100 and the threshold is $12,400. Its taxable income of $20,876 overstates the correct $20,175.59."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and nonrefundable credits eliminate the tax, but $36,276 − $16,100 leaves $20,176 taxable. A childless single filer has no nonrefundable credits, so $2,173.07 of tax remains."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed nonrefundable credits fully offset the tax without naming any, but a single filer with no dependents, childcare costs, or education costs has none. The $2,173.07 tax on $20,175.59 of taxable income stands."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used $15,750, the 2025 OBBBA standard deduction, instead of the 2026 inflation-adjusted $16,100. That overstates taxable income by $350 and gives $2,215 instead of $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It reduced the stated gross wages by the $21,208 employer-sponsored insurance premiums to $15,068 and concluded that no tax was owed. AGI is the full $36,276 in wages, and $20,175.59 remains taxable after the $16,100 deduction."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized by treating the $21,208 employer-sponsored insurance premiums as deductible medical expenses above the 7.5% floor. Employer-plan premiums are not out-of-pocket medical expenses, so actual itemized deductions are only $1,541.71, and the $16,100 standard deduction applies, giving $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction from 2025 before OBBBA instead of 2026's $16,100, along with outdated bracket thresholds. That inflates taxable income to about $21,276 and the tax to $2,352."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the employer-sponsored insurance premiums from wages, giving a $15,068 AGI, and it applied a post-sunset $8,535 deduction and $5,444 exemption. OBBBA made the TCJA permanent, so the full $36,276 AGI minus a $16,100 deduction is taxed at 10%/12%."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly kept AGI at $36,276 but used 2025 parameters, a $15,000 standard deduction and an $11,925 10% bracket threshold. The 2026 values are $16,100 and $12,400, and the correct tax is $2,173.07, not $2,315."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used an $8,000 deduction, a $5,000 exemption, and a 15% bracket above $11,600. OBBBA made the TCJA permanent, so 2026 has a $16,100 deduction, no exemption, and a 12% rate above $12,400."
+us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is nothing to score against the $2,173.07 from $20,175.59 of taxable income."
+us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It correctly found $20,176 of taxable income after the $16,100 deduction. It then said the EITC and other tax credits eliminate the liability, even though the EITC is refundable and excluded from this output, and a childless filer earning $36k is far above the EITC phase-out range."
+us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It used outdated parameters, a $15,750 deduction and a $12,150 10% bracket threshold, instead of $16,100 and $12,400, and got $2,220.12. It then submitted $424.76, which its own calculation does not support."
+us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction exceeds AGI, but $16,100 is far below $36,276 in wages. That leaves $20,175.59 of taxable income, and no nonrefundable credits exist to offset the $2,173.07 tax."
us,scenario_045,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified the childless EITC pathway and that its phaseout ends near $20,000, but then awarded $632 at $36,276. The childless EITC is fully phased out at that income, so this component is $0."
us,scenario_045,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from wages when computing EITC income, despite no stated pre-tax payroll treatment. It also produced $2,410 for a childless EITC whose maximum is far below that amount; using the full $36,276 leaves the credit fully phased out."
us,scenario_045,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated earned income as $15,068 by subtracting the $21,208 insurance premium from $36,276 of gross wages. The EITC test uses the full $36,276 here, at which the childless credit is fully phased out."
@@ -3918,127 +3988,130 @@ us,scenario_045,snap,claude-opus-4.7,llm_error,state_local_rule,False,"It claime
us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so nothing substantive can be graded. The correct answer is $0: gross income of $3,022.97 a month is 232% FPL, above both the 130% gross limit and Michigan's 200% BBCE limit."
us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly found that $3,023 a month exceeds the 130% FPL gross limit and that the household is not eligible, which means $0. It submitted $7,032 anyway, a value its own explanation does not support. It also never checked Michigan's 200% FPL BBCE limit (which the household also fails), and it overstated the 130% limit as $1,766 when the guideline gives about $1,695 a month."
us,scenario_045,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model first computed the correct $1,291 using the $5,900 exemption. It then submitted $1,229, which equals 4.25% of $36,276 less the wrong $7,350 exemption it also floated. So it replaced the correct 2026 exemption with an inflated one."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a $5,800 personal exemption instead of Michigan's 2026 exemption of $5,900. Taxable income came out $100 too high ($30,476), and the tax was overstated by $4.25 at $1,295.23."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 employer insurance premium input from wages, which gave an AGI of $15,068. It then applied the federal $14,600 standard deduction, which Michigan does not allow, in place of the $5,900 personal exemption. Finally it treated the refundable Homestead Property Tax Credit as a nonrefundable credit to reach $0, when Michigan tax on $30,375.59 at 4.25% is $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a $5,800 personal exemption instead of the 2026 amount of $5,900. That gave taxable income of $30,476 and tax of $1,295 rather than $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own work gave $1,295.23 using a $5,800 exemption, which is already $100 below the correct $5,900. It then submitted $1,024, which nothing in its derivation supports. The final value contradicts its own calculation."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model used a $5,800 exemption instead of $5,900, which gave $1,295. It then subtracted about $60 of nonrefundable credits that do not exist for this filer to reach $1,235. A single childless wage earner has no Michigan nonrefundable credit, so tax before refundable credits is the full $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model subtracted a $5,800 personal exemption instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $100 and the tax by $4.25, giving $1,295.23."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% flat rate. It also used a $6,000 exemption instead of $5,900, giving $30,276 × 4.05% = $1,226.18 rather than $30,375.59 × 4.25% = $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed $1,295 with a $5,800 exemption, which is $100 below the correct $5,900. It then inflated the answer to $1,490 by citing a 'rounding' and 'exemption phase-in' adjustment that does not exist, so the submitted value contradicts its own arithmetic."
-us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $15,068, which means it subtracted the $21,208 employer insurance premium input from wages even though that input does not reduce wage income. It also used a $6,200 exemption instead of $5,900. Taxable income should be $30,375.59, not $8,868."
-us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $15,068 by subtracting the employer insurance premium input from wages, and used a $5,670 exemption instead of $5,900. That gave $399 instead of 4.25% of $30,375.59."
-us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $5,600 personal exemption instead of Michigan's 2026 exemption of $5,900. Taxable income came out $300 too high and the tax was $1,303.73 instead of $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $6,000 personal exemption instead of $5,900. That understated taxable income by $100 and gave $1,286.73 instead of $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model took Michigan AGI as $15,068 by subtracting the $21,208 employer insurance premium input from wages, when AGI is the full $36,275.59. It also used a $5,600 exemption instead of $5,900, which cut taxable income to $9,468."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no derivation. At 4.25%, its $1,150 implies taxable income of about $27,060, which means it subtracted roughly $9,200 from $36,276 instead of the $5,900 personal exemption."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income of $30,476 shows it used a $5,800 exemption instead of Michigan's 2026 exemption of $5,900. The result was $1,295 instead of $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a federal AGI of $15,068 by subtracting the employer insurance premium input from wages, and a $5,600 exemption instead of $5,900. That shrank taxable income to $9,468 instead of $30,375.59."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model claimed that 'exemptions and standard deductions' eliminate the tax. Michigan has no standard deduction for a 44-year-old, and its only offset is the $5,900 personal exemption, which leaves $30,375.59 taxable and $1,290.96 of tax."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $15,068 by subtracting the $21,208 employer insurance premium input from wages, and used a $5,600 exemption instead of $5,900. That gave $402.39 instead of 4.25% of $30,375.59."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted a $5,800 personal exemption instead of the 2026 amount of $5,900. That gave taxable income of $30,476 and tax of $1,295.23."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $5,800 Michigan personal exemption instead of $5,900. That overstated taxable income by $100 and gave $1,295.23."
-us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $20,000 Michigan standard deduction and subtracted the Homestead Property Tax Credit as if it were nonrefundable. That credit is refundable and does not reduce tax before refundable credits. The model then submitted $0 even though its own reasoning reached $121."
-us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a personal exemption of about $5,200 instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $700 and gave $1,321."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that Michigan's 'deduction structure' leaves no taxable income. The only offset is the $5,900 personal exemption, which leaves $30,375.59 taxed at 4.25% for $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed that standard deductions and nonrefundable credits fully offset the tax. Michigan allows only the $5,900 personal exemption and no nonrefundable credit here, so tax before refundable credits is $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% flat rate, and a $5,800 exemption instead of $5,900. That produced $30,476 × 4.05% = $1,234.28."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model had the right $5,900 exemption but started from an AGI of $15,068, subtracting the $21,208 employer insurance premium input from wages. AGI is the full $36,275.59 of wages, so taxable income is $30,375.59, not $9,168."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed a Michigan standard deduction plus exemptions offset all $36,276 of income. Michigan has no standard deduction for this filer and only a $5,900 personal exemption, which leaves $30,375.59 taxable."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a federal AGI of $15,068 by subtracting the employer insurance premium input from wages, and a projected $5,880 exemption instead of $5,900. That gave taxable income of $9,188 instead of $30,375.59."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model projected a 2026 personal exemption of $5,800 instead of the published $5,900. That gave taxable income of $30,476 and tax of $1,295."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model subtracted a $6,000 personal exemption instead of $5,900. That understated taxable income by $100 and gave $1,286.73."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $900 and gave $1,329."
-us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model subtracted about $1,159 of Homestead Property Tax Credit from its $1,287 tax, treating it as nonrefundable. That credit is refundable and excluded from tax before refundable credits. It also used a $6,000 exemption instead of $5,900."
-us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $1,290.96."
-us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model allowed a $6,000 personal exemption instead of Michigan's 2026 exemption of $5,900. That gave taxable income of $30,276 and tax of $1,286.73."
-us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model computed about $1,329 using a $5,000 exemption, $900 below the correct $5,900. It also correctly noted that the Homestead credit is refundable and excluded. It then submitted $0 anyway, which contradicts its own derivation."
-us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption instead of $5,900. That gave taxable income of $31,276 and tax of $1,329.23."
-us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's derivation gave $1,303.73 using a $5,600 exemption, $300 below the correct $5,900. It then submitted $1,116.01, which nothing in its reasoning supports, so the final value contradicts its own calculation."
-us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted a standard deduction and the $5,200 of child support paid to reach $0. Michigan allows neither: child support paid is not deductible and there is no standard deduction for this filer. Only the $5,900 exemption applies, which leaves $30,375.59 taxable."
-us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said the homestead property tax credit needs home ownership and so excluded the renter. In fact Michigan treats 23% of rent as property tax, which gives a credit of 60% × ($2,428.80 − 3.2% × $36,276) = $760.79."
-us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Worked out the homestead credit correctly as $760.78: 23% of rent, minus 3.2% of resources, times 60%. It then submitted $884 citing an unexplained 'rounding' adjustment that nothing in its own working supports."
-us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Considered only the EITC and the Home Heating Credit, concluded both were $0, and then submitted an unsupported $1,209. It never computed the renter's homestead property tax credit of 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Counted only the Michigan EITC, which is $0, and left out the refundable Homestead Property Tax Credit. That credit is worth $760.79 to this renter, based on 23% of $10,560 rent."
-us,scenario_045,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Treated 20% of rent ($2,112) as property tax instead of Michigan's 23% for renters whose heat isn't included ($2,428.80). This gave 60% × ($2,112 − $1,160.83) = $570.70 instead of $760.79."
-us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Concluded the homestead credit didn't apply because no property tax was listed. It overlooked that Michigan counts 23% of the $10,560 rent as property tax, which gives a $760.79 refundable credit."
-us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Correctly found federal and Michigan EITC to be $0, then made up $1,089 from a 'reduced federal EITC estimate'. It never computed the homestead property tax credit, which is the only refundable credit here, at $760.79."
-us,scenario_045,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Got all three homestead parameters wrong: 20% of rent instead of 23%, a 3.5% income threshold instead of 3.2%, and a guessed ~55% rate instead of 60%. This gave $460 instead of 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Considered only the Michigan EITC, which is $0, and said no other refundable credits apply. It left out the renter's refundable Homestead Property Tax Credit of $760.79."
-us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Added a Michigan EITC based on a $342 federal EITC, but a childless filer earning $36,276 gets no federal EITC. It also cut household resources to $15,068 by subtracting the ESI premiums and used 20% of rent instead of 23%, which overstated the homestead credit at about $978."
-us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and an 80% credit rate instead of 60%. This gave $673.87 instead of $760.79."
-us,scenario_045,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, and never applied the 60% credit percentage to the excess over 3.2% of resources. It submitted the raw excess of $951.17 rather than 60% × $1,267.98 = $760.79."
-us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Cut total household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums, so the 3.2% threshold fell to $482.18. The reference uses resources of $36,275.59, which gives 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Concluded that no Michigan refundable credits apply and left out the Homestead Property Tax Credit. That credit gives this renter $760.79: 60% of 23% of rent minus 3.2% of resources.
-us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Considered only the Michigan EITC, which is $0, and never computed the refundable Homestead Property Tax Credit of $760.79 for the renter."
-us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Added a $62.10 Michigan EITC from a nonexistent $207 federal EITC and used 20% of rent instead of 23%. Its $977.89 homestead figure also implies household resources of $15,068, with the ESI premiums wrongly subtracted, rather than $36,276."
-us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported $0 and left out the refundable Homestead Property Tax Credit. For this renter that credit is 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Gave a childless filer with $36,276 of wages a $338.80 federal EITC, although the childless EITC phases out below that income, and took 30% of it. It also left out the $760.79 homestead property tax credit entirely."
-us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $695.48 equals 60% × ($2,428.80 − 3.5% × $36,276), so it used a 3.5% household-resource threshold instead of 3.2%. With 3.2% the credit is $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, which lowered the 3.2% threshold to $994.43. The reference keeps resources at $36,275.59, which gives $760.79."
-us,scenario_045,state_refundable_credits,glm-5.2,llm_error,other,False,"Computed a $570.70 homestead credit, itself understated because it used 20% of rent instead of 23%, and then submitted $0, contradicting its own working. The correct figure is 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Said no refundable Michigan credits apply beyond the $0 EITC, leaving out the renter's $760.79 Homestead Property Tax Credit."
-us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable Michigan credits, overlooking the Homestead Property Tax Credit. That credit treats 23% of the $10,560 rent as property tax and yields $760.79."
-us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Assumed that a Michigan refundable credit requires a dependent or some other unlisted condition. The Homestead Property Tax Credit needs only rent paid and resources under the limit, giving $760.79."
-us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Treated 20% of rent as property tax instead of 23%. That gave 60% × ($2,112 − $1,160.83) = $570.70 instead of 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the ESI premiums, which inflated the homestead credit to $1,168. It also added a $104 state EITC, although federal EITC is $0 for a childless filer at $36,276."
-us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,Treated Michigan's refundable credits as unavailable to a childless household. The Homestead Property Tax Credit has no children requirement and gives this renter $760.79.
-us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Deducted the $21,208 insurance premiums from household resources, giving $15,068 and a 3.2% threshold of only $482.18. The reference uses $36,275.59 of resources, which gives $760.79."
-us,scenario_045,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and left out the 60% credit percentage. It submitted the raw $951 excess instead of 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from household resources, giving $31,076, which lowered the 3.2% threshold. The reference keeps resources at $36,275.59, which gives $760.79."
-us,scenario_045,state_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 insurance premiums from household resources, giving $15,068, and used 20% of rent instead of 23%, producing $977.89. The correct figure is 60% × ($2,428.80 − 3.2% × $36,275.59) = $760.79."
-us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no state refundable credits qualify, leaving out the renter's refundable Homestead Property Tax Credit of $760.79."
-us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Calculated the homestead credit on $15,068 of resources (with ESI premiums subtracted) and a 3.5% threshold, giving $1,141. It then added a 6% Michigan EITC on a nonexistent $341 federal EITC, when the credit is 60% × ($2,428.80 − $1,160.82) = $760.79 and EITC is $0."
-us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, which inflated the credit to $861. The reference uses resources of $36,275.59, which gives $760.79."
-us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Reported no Michigan refundable credits and left out the Homestead Property Tax Credit. That credit is 60% of 23% of rent minus 3.2% of resources, which is $760.79."
-us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the Homestead Property Tax Credit as limited by tax liability, when it is fully refundable and paid regardless of liability. It also used an outdated 6% EITC rate, though the EITC is $0 either way. The credit is $760.79."
-us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so nothing could be scored against the $760.79 homestead credit."
-us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and a 3.5% resource threshold instead of 3.2%. This gave 60% × ($2,112 − $1,269.66) = $505.40 instead of $760.79."
-us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Wrongly treated the listed $10,560 pre-subsidy rent as not actually paid and so denied the homestead credit. At 23% of that rent, $2,428.80 exceeds 3.2% of resources by $1,267.98, and 60% of that is $760.79."
-us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used 17% of rent as property tax instead of 23% and a 3.5% threshold instead of 3.2%. This gave $315.32 instead of 60% × ($2,428.80 − $1,160.82) = $760.79."
-us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Concluded the homestead property tax credit did not qualify on the listed facts. The $10,560 of rent paid is enough, since 23% of it counts as property tax, and the credit is $760.79."
-us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Said no refundable state credits apply, leaving out the renter's refundable Michigan Homestead Property Tax Credit of $760.79."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly computed 0.0425 × ($36,276 − $5,900) ≈ $1,291 and then dropped it. Its later guesses about a higher exemption ($6,350 or $7,350) are unfounded, and the $1,229 it submitted does not follow from any of the figures it stated."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900. That left taxable income $100 too high and produced $1,295.23 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied the $14,600 federal standard deduction, which Michigan does not use (Michigan allows only the $5,900 personal exemption), and subtracted the $21,208 ESI premiums from AGI, which equals the full $36,276 in wages. It also treated the Homestead Property Tax Credit as nonrefundable, which zeroed out a tax that is actually $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used a $5,600 exemption instead of $5,900, which gave about $1,303. It then submitted $1,280, a number its own arithmetic does not support. The correct result is ($36,276 − $5,900) × 4.25% = $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of the 2026 amount of $5,900, so taxable income came out at $30,476 rather than $30,376. That produced $1,295 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning computed $1,295 using a $5,800 exemption (already $100 short of the correct $5,900), but it submitted $1,024. Nothing in its reasoning produces that number."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $5,800 exemption instead of $5,900 and then subtracted about $60 of made-up nonrefundable credits. No Michigan nonrefundable credit applies to this childless filer, so the tax is $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It subtracted a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900. That produced $1,295.23 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 4.05%, the one-time 2023 reduced rate, instead of the 4.25% rate that applies in 2026. It also projected a $6,000 exemption instead of $5,900, which gave $1,226.18 rather than $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $1,295 with a $5,800 exemption and then raised it to $1,490 by citing an 'exemption phase-in' that does not exist. Michigan's exemption does not phase out at this income, and the correct tax is ($36,276 − $5,900) × 4.25% = $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages and arrived at an AGI of $15,068. The actual AGI is the full $36,276 in wages. It also used a $6,200 exemption instead of $5,900, which gave $376.89."
+us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $15,068 by subtracting the $21,208 ESI premiums, but AGI equals the full $36,276 in wages. It also used a $5,670 exemption instead of $5,900, which gave $399 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $5,600 personal exemption, an outdated amount, instead of the 2026 amount of $5,900. That overstated taxable income by $300 and gave $1,303.73."
+us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It projected a $6,000 personal exemption instead of the published 2026 amount of $5,900. That understated taxable income by $100 and gave $1,286.73 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums and took AGI as $15,068, but AGI equals the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It showed no derivation. At 4.25%, its $1,150 implies taxable income of about $27,059, which means an exemption of roughly $9,200 instead of $5,900. The correct result is $30,375.59 × 4.25% = $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income of $30,476 reflects a $5,800 exemption instead of the 2026 amount of $5,900. That produced $1,295 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a 'Federal AGI' of $15,068 by subtracting the $21,208 ESI premiums, but AGI is the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed exemptions and 'standard deductions' eliminate the tax, but Michigan has no standard deduction for a 44-year-old filer. Only the $5,900 personal exemption applies, which leaves $30,375.59 taxed at 4.25% for $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used $15,068 as AGI after subtracting the $21,208 ESI premiums, but AGI equals the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted a $5,800 personal exemption instead of the 2026 amount of $5,900, which produced $1,295.23 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900, so taxable income came out at $30,476 and tax at $1,295.23 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 Michigan standard deduction and subtracted the Homestead Property Tax Credit as if it were nonrefundable, but that credit is refundable and does not count toward this output. It then submitted $0, which contradicts its own $121 figure. The only reduction is the $5,900 exemption, which gives $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $5,200 personal exemption instead of the 2026 amount of $5,900. That overstated taxable income by $700 and gave $1,321."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed Michigan's deduction structure leaves no taxable income, but Michigan allows only a $5,900 personal exemption here. That leaves $30,375.59 taxed at 4.25% for $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed made-up standard deductions and nonrefundable credits would fully offset the tax. Michigan has no standard deduction for this filer and no applicable nonrefundable credit, so only the $5,900 exemption applies and the tax is $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It applied the 2023-only 4.05% rate instead of the 4.25% rate for 2026. It also used a $5,800 exemption instead of $5,900, which gave $1,234.28."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used the correct $5,900 exemption but started from an AGI of $15,068 after subtracting the $21,208 ESI premiums. AGI is the full $36,276 in wages, so the tax is $1,290.96, not $390."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed a Michigan standard deduction and exemptions offset the full $36,276. In fact Michigan allows only the $5,900 personal exemption for this filer, which leaves $30,375.59 taxable and $1,290.96 of tax."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It took federal AGI as $15,068 by subtracting the $21,208 ESI premiums, but AGI is the full $36,276 in wages. It also projected a $5,880 exemption instead of $5,900, which gave $390."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It projected a $5,800 personal exemption instead of the 2026 amount of $5,900, which produced $1,295 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $6,000 personal exemption instead of the 2026 amount of $5,900, which understated taxable income by $100 and gave $1,286.73."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a stale $5,000 personal exemption instead of the 2026 amount of $5,900. That overstated taxable income by $900 and gave $1,329."
+us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted the Homestead Property Tax Credit (about $1,159) from tax before refundable credits, but that credit is refundable and is excluded from this output. It also used a $6,000 exemption instead of $5,900. The correct pre-refundable tax is $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits. The correct derivation is ($36,275.59 − $5,900) × 4.25% = $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used a $6,000 personal exemption instead of Michigan's 2026 amount of $5,900, which gave $1,286.73 instead of $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It computed about $1,329 using a stale $5,000 exemption, correctly called the homestead credit refundable, and then submitted $0 with no supporting step. The correct tax with the $5,900 exemption is $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used an outdated $5,000 personal exemption instead of the 2026 amount of $5,900, so taxable income came out at $31,276 and tax at $1,329.23."
+us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $1,303.73 with a $5,600 exemption ($300 short of the correct $5,900) and then submitted $1,116.01, which nothing in its reasoning supports. The correct result is $1,290.96."
+us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a standard deduction and the $5,200 child support payment. Michigan has no standard deduction for this filer, and child support paid is not deductible. Only the $5,900 exemption applies, which leaves $1,290.96 of tax."
+us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Wrongly treated the Michigan Homestead Property Tax Credit as limited to homeowners. Renters qualify, with 23% of the $10,560 rent ($2,428.80) counted as property tax, which yields a $760.79 refundable credit."
+us,scenario_045,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Looked only at the Michigan EITC, which is correctly $0, and never applied the refundable Homestead Property Tax Credit for renters. That credit is 60% × (23% of $10,560 rent − 3.2% of $36,276 resources) = $760.79."
+us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own reasoning computed the homestead credit correctly at about $760.78 (23% of rent, minus 3.2% of $36,276, times 60%). It then submitted $884, which nothing in its derivation supports."
+us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Considered only the Michigan EITC and the Home Heating Credit, concluded both were $0, and never applied the renter Homestead Property Tax Credit. It then submitted an unsupported $1,209 instead of the $760.79 homestead credit."
+us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Counted only the Michigan EITC, which is $0, and left out the refundable Homestead Property Tax Credit for renters. That credit is worth $760.79 here: 60% × ($2,428.80 − $1,160.82)."
+us,scenario_045,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Used the outdated 20% rent-to-property-tax factor ($2,112) instead of 23% ($2,428.80). The rest of its method (3.2% threshold, 60% rate) was correct, so it got $570.70 instead of $760.79."
+us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Concluded the homestead credit did not apply because no property tax was listed. It missed that renters claim it on 23% of rent paid ($10,560), which gives a $760.79 refundable credit."
+us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Correctly found federal and Michigan EITC to be $0, then made up a $1,089 EITC-based figure. It never applied the renter Homestead Property Tax Credit, which is the only refundable credit here ($760.79)."
+us,scenario_045,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and a guessed ~55% rate instead of the 60% rate that applies at $36k of resources. That produced $460 instead of $760.79."
+us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Counted only the zero Michigan EITC and wrongly stated that no other refundable credits apply. The renter Homestead Property Tax Credit applies and equals $760.79.
+us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the $21,208 employer-sponsored premiums, used 20% of rent instead of 23%, and added a $102.60 Michigan EITC on a made-up $342 federal EITC. A childless filer with $36k of wages gets no federal EITC, and resources stay at $36,276."
+us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and an 80% credit rate instead of 60%. The correct figure is 0.6 × ($2,428.80 − $1,160.82) = $760.79."
+us,scenario_045,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and never applied the 60% credit percentage, so it reported the raw $951.17 excess as the credit. The correct figure is 60% × $1,267.98 = $760.79."
+us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Applied the 23% rent factor and 60% rate correctly but cut total household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums. Resources stay at the full $36,276, so the threshold is $1,160.82 and the credit is $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Returned $0 and missed the Michigan Homestead Property Tax Credit, which is refundable for renters. With $10,560 rent and $36,276 in resources it equals $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Looked only at the Michigan EITC, which is $0, and left out the refundable renter Homestead Property Tax Credit of 60% × (23% × $10,560 − 3.2% × $36,276) = $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $977.89 homestead figure comes from using 20% of rent instead of 23% and from cutting resources to $15,068 by subtracting the employer premiums. It also added a $62.10 Michigan EITC on a made-up $207 federal EITC that a childless filer at $36k cannot receive."
+us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for $10,560 rent and $36,276 in resources."
+us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Made up a $338.80 federal EITC for a childless filer whose $36,276 in wages is far past the phaseout, so the Michigan EITC is really $0. It also left out the renter Homestead Property Tax Credit of $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $695.48 equals 0.6 × ($2,428.80 − 3.5% × $36,276), so it subtracted 3.5% of household resources instead of 3.2%. The correct subtraction is $1,160.82, which gives $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076. Child support paid is not a reduction, so resources stay at $36,276, the 3.2% threshold is $1,160.82, and the credit is $760.79."
+us,scenario_045,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% to compute $570.70, then submitted $0 anyway, contradicting its own conclusion that the credit applies. The correct credit is 60% × ($2,428.80 − $1,160.82) = $760.79."
+us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,Counted only the zero Michigan EITC and wrongly said no other refundable credits apply. It missed the $760.79 renter Homestead Property Tax Credit.
+us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no Michigan refundable credit even though $10,560 in rent was listed. Renters qualify for the Homestead Property Tax Credit, worth $760.79 here."
+us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,Assumed Michigan refundable credits require a dependent. The Homestead Property Tax Credit has no dependent requirement and pays this single renter $760.79.
+us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used the 20% rent factor ($2,112) instead of 23% ($2,428.80), so the 60% credit came to $570.70 instead of $760.79."
+us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the $21,208 employer premiums, which inflated the homestead credit to $1,168. It also added a $104 state EITC even though the childless filer's federal EITC is $0."
+us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,Assumed a childless household has no Michigan refundable credit. The renter Homestead Property Tax Credit applies without children and equals $760.79.
+us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Treated the $21,208 employer-sponsored insurance premiums as a reduction to household resources, giving $15,068. Resources stay at $36,276, so the 3.2% threshold is $1,160.82 and the credit is $760.79, not $1,167.97."
+us,scenario_045,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and never applied the 60% credit percentage, so it submitted the raw $951 excess. The correct credit is 60% × $1,267.98 = $760.79."
+us,scenario_045,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from household resources, giving $31,076. That shrank the 3.2% threshold to $994.43 and inflated the credit to $860.62, when resources of $36,276 give $760.79."
+us,scenario_045,state_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the employer insurance premiums and used 20% of rent instead of 23%, giving $977.89. The correct calculation is 60% × ($2,428.80 − $1,160.82) = $760.79."
+us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for this household."
+us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Cut resources to $15,068 by subtracting the employer premiums and used a 3.5% threshold instead of 3.2%, giving a $1,141 homestead credit. It also added a $20 EITC at the outdated 6% match on a made-up $341 federal EITC, when the childless filer's federal EITC is $0."
+us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, so the threshold came out too low. With resources of $36,276 the credit is $760.79, not $861."
+us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Homestead Property Tax Credit available to Michigan renters, worth $760.79 here."
+us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Wrongly treated the Homestead Property Tax Credit as limited to tax liability. It is fully refundable, so the whole $760.79 counts as a state refundable credit. It also used the outdated 6% EITC match."
+us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer or explanation for state_refundable_credits, so nothing could be scored against the $760.79 homestead credit."
+us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and a 3.5% resource threshold instead of 3.2%, giving $505.40. The correct figure is 0.6 × ($2,428.80 − $1,160.82) = $760.79."
+us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Wrongly treated the listed pre-subsidy rent as not actually paid and decided the renter credit was not triggered. With no subsidy listed, the full $10,560 is rent paid, and 23% of it exceeds 3.2% of resources, producing a $760.79 credit."
+us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used 17% of rent instead of 23% and a 3.5% threshold instead of 3.2%, giving $315.32. The correct figure is 60% × ($2,428.80 − $1,160.82) = $760.79."
+us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Wrongly decided the listed facts do not qualify for the homestead credit. A renter paying $10,560 with $36,276 in resources gets 60% of the amount by which 23% of rent exceeds 3.2% of resources, which is $760.79."
+us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for this household."
us,scenario_046,child1_chip_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied a generic 255% FPL CHIP limit that is not Oklahoma's rule. In Oklahoma a 10-year-old falls outside the separate CHIP's age coverage and gets children's coverage only through SoonerCare Medicaid, whose income test the household also fails. It also understated MAGI as $77,164 by subtracting the $14,839 FLSA overtime premium ($92,003 - $14,839), but the overtime deduction does not reduce AGI-based MAGI. Correct MAGI is about $92,003, well above Oklahoma's children's limit, so the answer is not eligible."
us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model asserted that the child was “likely eligible” without testing Oklahoma's income limits or any categorical pathway. At 2.79 times FPL, child1 fails the regular children's Medicaid limit, CHIP limit, and all other applicable categories, yielding medicaid_category = NONE and value = 0."
us,scenario_046,child2_chip_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's MAGI of $77,164 is exactly $92,003 minus the $14,839 FLSA overtime premium. It treated the no-tax-on-overtime deduction as lowering MAGI, but that deduction is taken below the line and never reduces AGI or Medicaid/CHIP MAGI. Using the correct MAGI of $92,003, the four-person household sits at about 280% FPL. That is above even the 255% CHIP limit the model cited, so child2 is not CHIP-eligible."
us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that being a child made Child 2 eligible without testing Oklahoma's categorical and income requirements. At 2.79 times FPL, Child 2 exceeds all applicable child Medicaid limits and qualifies through no other pathway, so the correct value is 0."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got close to the right answer. It applied the overtime deduction and the $4,400 CTC and reached about $488, using a $32,300 standard deduction instead of $32,200. It then submitted $3,465 after an unexplained 'adjustment' that none of its own steps support."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a $28,700 standard deduction instead of $32,200 and left out the $14,839 overtime deduction, which inflated taxable income to $63,303. It also applied a $2,500 AOTC even though no qualified tuition expenses were listed, and used a $2,000-per-child CTC instead of $2,200."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the OBBBA overtime deduction, used a $31,500 standard deduction and a $2,000-per-child CTC, and so reached $2,777 in its reasoning. It then submitted $4,669, which none of its own steps produce. The correct figure uses taxable income of $44,964 and a $4,400 CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It left out the $14,839 overtime deduction, which gave it taxable income of $59,703. It then submitted about the pre-credit tax ($6,779) without subtracting any nonrefundable CTC, even though its own reasoning said the credits reduce the tax to near zero."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It correctly got taxable income of $44,964 from the $32,200 standard deduction plus the overtime deduction. It then reported the tax before child credits ($4,966, using a $24,000 10% bracket instead of $24,800) and never subtracted the $4,400 nonrefundable CTC, which the output definition requires."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the OBBBA qualified overtime deduction for the $14,839 FLSA premium. It also used the 2025 $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 and $24,800 figures, and a $2,000-per-child CTC instead of $2,200, arriving at $2,963."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It dismissed the FLSA overtime premium as 'informational' and so never took the qualified overtime deduction. It then submitted $12,200, which is higher than its own pre-credit tax of $6,696, with no nonrefundable CTC subtracted."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction correctly but itemized at about $40.9k by counting health insurance premiums as medical expenses above the 7.5% floor. The household's deductible itemized total is below the $32,200 standard deduction, which the reference uses. Its understated taxable income ($36.2k) made the tax smaller than the $4,400 CTC, so it answered $0 instead of $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized by counting $24,330 of health insurance premiums and medical costs as deductible medical expenses, when the standard deduction should apply. It also left out the $14,839 overtime deduction. Finally, it treated only $200 per child as the nonrefundable CTC, when the full $4,400 CTC is nonrefundable up to the tax liability."
-us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset for 2026, so it applied personal exemptions, pre-TCJA 10%/15% brackets and a $1,000-per-child CTC. OBBBA made the TCJA structure permanent: $32,200 standard deduction, 12% bracket and $2,200 CTC. It also left out the overtime deduction."
-us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law ($21,200 of personal exemptions, itemized deductions, pre-TCJA rates and a $2,000 total CTC) instead of the permanent OBBBA rules. Under those rules the $32,200 standard deduction plus the $14,839 overtime deduction gives $44,964 of taxable income and a $4,400 CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It got taxable income of $44,964 right. It then inflated the CTC to $2,260 per child ($4,520) instead of the statutory $2,200 ($4,400), and used a $24,400 10% bracket instead of $24,800. Together these understated the result at $387.68."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It added the overtime premium on top of wages (which already include it) and subtracted ESI premiums that were not listed as payroll deductions. It also assumed the TCJA had sunset ($5,000 personal exemptions, a $1,000-per-child CTC) and never took the OBBBA overtime deduction or the $32,200 standard deduction."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used a $31,200 standard deduction with no overtime deduction and a $4,000 CTC. Its $8,518 is also higher than the pre-credit tax those inputs produce under the 10%/12% brackets (about $6,800), so its arithmetic is internally wrong as well."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules (personal exemptions, itemizing, a $1,000-per-child CTC) and subtracted ESI premiums from wages. It never applied the OBBBA overtime deduction, the $32,200 standard deduction or the $2,200-per-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset (personal exemptions, a $1,000-per-child CTC), subtracted ESI premiums from AGI and left out the overtime deduction. It also claimed a $1,500 nonrefundable AOTC even though no qualified tuition expenses were listed, which makes the AOTC zero."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is taxable income of $44,964 (standard deduction plus the $14,839 overtime deduction), tax of $4,899.71, and the $4,400 CTC. Its $3,212 is several thousand dollars too high, which fits leaving out the overtime deduction and not applying the full $2,200-per-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its explanation mentions only standard/itemized deductions and the nonrefundable CTC, and never the qualified overtime deduction. Its $1,858.40 implies taxable income well above the correct $44,964, meaning it left out the $14,839 overtime deduction before applying the credits."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions and itemized deductions, which is TCJA-sunset logic. The permanent 2026 rules eliminate personal exemptions and apply the $32,200 standard deduction, the $14,839 overtime deduction and a $2,200-per-child CTC, leaving $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets, left out the $14,839 OBBBA overtime deduction, and applied a $2,000-per-child CTC instead of $2,200, giving $2,963.36."
-us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value or explanation for this output, so there was no answer to score."
-us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It got taxable income ($44,964) and tax ($4,899.68) right. It then treated only $500 per child ($2,200 minus the $1,700 refundable cap) as nonrefundable. In fact the full $4,400 CTC is nonrefundable up to the tax liability, and the ACTC cap applies only to the unused remainder."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It skipped the calculation and assumed the American Opportunity Credit wiped out the tax. No qualified tuition expenses were listed, so the AOTC is zero. The tax of $4,899.71 on $44,964 of taxable income, less the $4,400 CTC, leaves $499.71, not $0."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It applied an AOTC even though no tuition expenses were listed, left out the overtime deduction, and never subtracted the $4,400 nonrefundable CTC. As a result its $6,794 is roughly the tax before child credits on taxable income that excludes the overtime deduction."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It capped the qualified overtime deduction at the $12,500 single-filer limit instead of the $25,000 joint limit. That cut the deduction from $14,839 to $12,500 and raised taxable income to $47,303, giving $780.36 instead of $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. It cited an irrelevant 'AOTC phaseout' and submitted a round $8,500, which is higher than even the pre-credit tax. It never applied the $32,200 standard deduction, the $14,839 overtime deduction or the $4,400 nonrefundable CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset: four personal exemptions of about $5,460 each, 10%/15% brackets, and a $1,000-per-child CTC. It also left out the overtime deduction. The permanent 2026 rules use the $32,200 standard deduction and a $2,200-per-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly applied post-TCJA-sunset law ($21,168 of personal exemptions, itemized deductions, two $1,000 CTCs) and left out the OBBBA overtime deduction. OBBBA made the $32,200 standard deduction and the $2,200-per-child CTC permanent for 2026."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a 'post-TCJA' 2026 joint return with $21,200 of personal exemptions and a $1,000-per-child CTC. It never applied the permanent $32,200 standard deduction, the $14,839 qualified overtime deduction or the $2,200-per-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules, with $23,508 of taxable income and a $2,000 total CTC. The correct rules give $44,964 of taxable income (standard deduction plus overtime deduction) and a $4,400 nonrefundable CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction but itemized with about $19,193 of medical expenses by counting health insurance premiums above the 7.5% floor. The household's deductible itemized total is below the $32,200 standard deduction. That understated taxable income enough that the $4,400 CTC appeared to wipe out the tax, giving $0 instead of $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for this output, so there was no answer to score."
-us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted a placeholder $9,999 with no derivation. That figure is higher than even the pre-credit tax. It ignored the $32,200 standard deduction, the $14,839 overtime deduction and the $4,400 nonrefundable CTC, which together give $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction and the $2,200-per-child CTC correctly. It then itemized with $18,080 of medical expenses by counting $24,000 of health insurance premiums, when the deductible itemized total falls below the $32,200 standard deduction. That understated taxable income ($36,392), so the tax appeared to be fully offset and it answered $0."
-us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It rejected the overtime deduction on the grounds that wages already include the premium, but the OBBBA deduction applies to the premium included in wages. It also itemized using $24,000 of premiums as medical expenses, claimed a $1,000 AOTC with no qualified expenses, and used a $2,000-per-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left out the $14,839 overtime deduction, used a $31,500 standard deduction, and claimed a $2,500 AOTC even though no tuition expenses were listed. Its reasoning ended at $342, but it submitted $8,838, a figure none of its steps produce."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation was nearly right: taxable income of $44,864, tax of $4,888 less the $4,400 CTC, about $488. It then submitted $3,465 based on an unexplained 'adjustment', dropping its correct result. It also used a $32,300 standard deduction instead of $32,200."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and used a stale $28,700 MFJ standard deduction instead of $32,200, which inflated taxable income to $63,303. It also applied a $2,500 AOTC even though there were no qualified tuition expenses. It used a $2,000-per-child CTC instead of $2,200 and capped the CTC incoherently."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed without any computation that credits wipe out the tax. Taxable income is $44,964.29 after the $32,200 standard deduction and the $14,838.71 overtime deduction, which gives $4,899.71 of tax. That exceeds the $4,400 CTC, so $499.71 remains."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never applied the OBBBA qualified overtime deduction ($14,838.71) and used a $31,500 standard deduction instead of $32,200, which gave $60,503 of taxable income. It used a $2,000-per-child CTC instead of $2,200, reached $2,777, and then submitted an unrelated $4,669."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction, leaving $59,703 of taxable income. It then submitted $6,779, roughly its pre-credit tax, without subtracting the nonrefundable CTC, even though its own reasoning said credits bring the tax near zero."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It correctly reached $44,964 of taxable income but used a $24,000 top for the 10% bracket instead of the 2026 MFJ $24,800. It then reported pre-credit tax ($4,966) instead of subtracting the $4,400 nonrefundable CTC, which the output definition requires."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the OBBBA qualified overtime deduction of $14,838.71. It used 2025 parameters instead of 2026 ones: a $30,000 standard deduction instead of $32,200, a $23,850 top for the 10% bracket instead of $24,800, and a $2,000-per-child CTC instead of $2,200. This produced $62,003 of taxable income and $2,963 of tax."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the FLSA overtime premium as purely informational and so never took the $14,838.71 qualified overtime deduction. It then submitted $12,200, which exceeds even its own $6,696 pre-credit tax, instead of subtracting the $4,400 nonrefundable CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It itemized at about $40.9k by double-counting the $12,000 health insurance premium, which is one amount reported under two labels. Correct itemized deductions are about $28.7k, below the $32,200 standard deduction. Taxable income is therefore $44,964.29, tax is $4,899.71, and $499.71 remains after the CTC instead of $0."
+us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the qualified overtime deduction and double-counted premiums as $24,330 of medical costs, so it wrongly itemized. It also treated only $200 per child of the CTC as nonrefundable. In fact the whole $2,200-per-child CTC is nonrefundable up to the tax liability, and only the excess is refundable as the ACTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law: personal exemptions, a 15% second bracket and a $1,000-per-child CTC. OBBBA made the TCJA rates, the $32,200 standard deduction and the $2,200 CTC permanent. It also omitted the $14,838.71 qualified overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed under expired pre-TCJA rules, with personal exemptions and a $1,000-per-child CTC, instead of OBBBA's 2026 regime: the $32,200 standard deduction, 10%/12% brackets and a $2,200 CTC. It also skipped the $14,838.71 qualified overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its taxable income of $44,964 was right, but it ended the 10% bracket at $24,400 instead of the 2026 MFJ $24,800. It also used an inflated $2,260-per-child CTC instead of $2,200, which netted $387.68 instead of $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added the overtime premium on top of wages even though gross wages already include overtime. It subtracted ESI premiums from AGI, which no input supports, and used pre-TCJA personal exemptions and a $1,000-per-child CTC. It also never applied the $14,838.71 overtime deduction or the $32,200 standard deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It omitted the qualified overtime deduction, used a $31,200 standard deduction and a $4,000 CTC, and submitted $8,518. That figure is larger than any tax its stated inputs produce. The correct taxable income is $44,964.29, giving $4,899.71 of tax less $4,400 of CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (personal exemptions and a $1,000-per-child CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It subtracted ESI premiums from AGI with no basis, and omitted the $14,838.71 qualified overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used pre-TCJA personal exemptions, a $1,000-per-child CTC and an ESI-premium subtraction from AGI. It also claimed a $1,500 nonrefundable AOTC with no qualified tuition expenses, and omitted the $14,838.71 overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,212 is consistent with taxing AGI less only a standard deduction and omitting the $14,838.71 qualified overtime deduction. The correct path gives $44,964.29 of taxable income, $4,899.71 of tax, and $499.71 after the $4,400 CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no usable derivation. $1,858.40 does not reflect the correct path: the $32,200 standard deduction plus the $14,838.71 overtime deduction, giving $44,964.29 of taxable income and $4,899.71 of tax. From that, the full $4,400 nonrefundable CTC leaves $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its reference to personal exemptions shows it applied pre-TCJA sunset rules instead of OBBBA's permanent 2026 regime of a $32,200 standard deduction and a $2,200 CTC. It also omitted the $14,838.71 qualified overtime deduction, producing $2,722.40."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction. It used 2025 values instead of 2026 ones: a $30,000 standard deduction instead of $32,200, a $23,850 top for the 10% bracket instead of $24,800, and a $2,000-per-child CTC instead of $2,200. The result was $2,963.36."
+us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No answer or reasoning was returned, so there is no parseable value for this output."
+us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It computed $4,899.68 of pre-credit tax correctly, then treated only $500 per child ($2,200 minus the $1,700 ACTC cap) as nonrefundable. The full $4,400 CTC is nonrefundable up to the tax liability, and the ACTC applies only to the unused excess. The answer should have been $499.71, not $3,899.68."
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It assumed the American Opportunity Credit offsets the tax even though no qualified tuition expenses are listed, so AOTC is $0. It never computed a liability. The correct result is $4,899.71 of tax less the $4,400 CTC, or $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and applied an AOTC with no qualified expenses. It also did not subtract the $4,400 nonrefundable CTC, and reported $6,794, which is roughly the tax with no overtime deduction and no CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It capped the qualified overtime deduction at $12,500, the single-filer limit, instead of the $25,000 joint-return limit. That cut the deduction from $14,838.71 and left $47,303 of taxable income instead of $44,964.29."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It invoked an irrelevant AOTC phaseout, omitted the $14,838.71 overtime deduction, and failed to subtract the $4,400 nonrefundable CTC, reporting an unsupported $8,500. The correct result is $4,899.71 of tax on $44,964.29 of taxable income, less $4,400."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law (personal exemptions, a 15% bracket and a $1,000-per-child CTC) instead of OBBBA's permanent 2026 rules. It also omitted the $14,838.71 overtime deduction and itemized instead of taking the larger $32,200 standard deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset, using personal exemptions and $1,000-per-child CTCs, even though OBBBA kept the $32,200 standard deduction, the 10%/12% brackets and the $2,200 CTC for 2026. It also never applied the $14,838.71 qualified overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It computed a 'post-TCJA' return with personal exemptions and a $1,000-per-child CTC instead of OBBBA's permanent 2026 parameters. It also omitted the $14,838.71 qualified overtime deduction."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA rules, with personal exemptions and a $2,000 total CTC, and reached $23,508 of taxable income. The correct figures are $44,964.29 of taxable income after the $32,200 standard deduction and the $14,838.71 overtime deduction, with a $4,400 CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction correctly but inflated the medical deduction to about $19,193 by double-counting the $12,000 premium, which is one amount under two labels. That made itemizing appear to beat the $32,200 standard deduction. With the standard deduction, tax is $4,899.71, and $499.71 remains after the $4,400 CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or reasoning was returned, so there is no parseable value for this output."
+us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted a $9,999 placeholder with no derivation. The correct result is $499.71: the standard and overtime deductions give $44,964.29 of taxable income and $4,899.71 of tax, less the $4,400 CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted medical expenses as $24,980 by double-counting the $12,000 premium and including over-the-counter costs, which produced $40,772 of itemized deductions. Correct itemized deductions fall below the $32,200 standard deduction. Taxable income is therefore $44,964.29 and tax is $4,899.71, which the $4,400 CTC does not fully offset."
+us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and double-counted the $12,000 premium, plus OTC costs, to justify itemizing. It also claimed a $1,000 AOTC despite no qualified expenses and used a $2,000-per-child CTC instead of $2,200."
+us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a full $2,500 AOTC even though no qualified tuition expenses exist, and omitted the $14,838.71 overtime deduction. It then submitted $8,838, which contradicts its own $342 result."
us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated the refundable CTC as the $1,700-per-child cap ($3,400) and never applied the rule that only CTC left after tax is refundable. Tax before credits (about $4,900) uses up the whole CTC, so refundable CTC is $0. Its submitted $3,200 does not even match its own $3,400 figure."
us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly zeroed the EITC and found the CTC fully used against tax. It then assumed the head had enough tuition to max out the AOTC and claimed its $1,000 refundable part. No qualified tuition and related expenses are listed, so the AOTC and its refundable 40% are both $0."
us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It counted the $4,400 CTC as refundable along with an assumed $1,000 AOTC and reached $4,966 of credits offsetting tax. That mixes nonrefundable credits with refundable ones. The CTC is fully used against tax of about $4,900, leaving $0 refundable CTC. The AOTC is $0 because no qualified tuition expenses are listed."
@@ -4054,69 +4127,71 @@ us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,miss
us,scenario_046,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It put tax before credits at $3,871.01, below the $4,400 CTC, and refunded the $528.99 remainder. Tax is higher: even after the $32,200 standard deduction and the $14,839 overtime deduction, tax on $92,003 is about $4,900. That uses up the whole CTC, so refundable CTC is $0."
us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It correctly found refundable CTC and EITC are $0. It then assumed a $1,000 refundable AOTC with no tuition listed, and submitted $1,500, which contradicts its own stated $1,000 total. With no qualified tuition and related expenses the AOTC is $0, so the correct total is $0."
us,scenario_046,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed $5,704 of Social Security tax and $1,334 of Medicare tax but then added them incorrectly as $10,038 and invented a $518.40 Oklahoma payroll-tax adjustment. Oklahoma contributes no mandatory employee payroll tax here, so the two federal components total $7,038."
-us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from FICA wages. The facts do not identify that premium as a pre-tax employee payroll deduction, so Social Security and Medicare apply to the full $92,000 of wages and total $7,038."
-us,scenario_046,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model added the $14,839 FLSA overtime premium to the stated $92,000 gross wages even though the prompt says gross wages already include all overtime, then also deducted the $9,208 insurance premium without a stated pre-tax payroll exclusion. FICA applies to the listed $92,000 wage total, yielding $7,038."
-us,scenario_046,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced FICA-taxable wages from $92,000 to $82,792 by treating the employer-sponsored insurance premium as a pre-tax employee payroll deduction. Applying 6.2% Social Security and 1.45% Medicare to the full wage amount yields $7,038."
-us,scenario_046,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $9,208 employer-sponsored insurance premium from the Social Security and Medicare wage base. No pre-tax payroll deduction is specified, so the full $92,000 is subject to FICA and produces $7,038."
-us,scenario_046,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the $9,208 insurance premium as a pre-tax exclusion and assessed FICA on only $82,792. The payroll-tax wage base is the full $92,000, producing $5,704 of Social Security tax and $1,334 of Medicare tax."
-us,scenario_046,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model wrongly excluded the $9,208 employer-sponsored insurance premium from FICA wages and therefore taxed only $82,792. Social Security and Medicare apply to the full $92,000, totaling $7,038."
-us,scenario_046,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so the required value was missing."
-us,scenario_046,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model set payroll tax to zero because no Oklahoma payroll-tax rate was supplied, ignoring the directly computable federal employee Social Security and Medicare taxes. Those federal components are $5,704 and $1,334, respectively, regardless of the absence of a state payroll tax."
-us,scenario_046,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated the correct federal tax categories but submitted $7,172 instead of applying their rates to the $92,000 wage base. The resulting Social Security and Medicare amounts are $5,704 and $1,334, with no Oklahoma employee payroll-tax addition, for $7,038."
-us,scenario_046,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model made a $10 arithmetic error in calculating employee Social Security and Medicare tax on the reported wages. The components are $5,704 and $1,334, which sum to $7,038 rather than $7,028."
-us,scenario_046,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages to $82,792 by subtracting the $9,208 insurance premium as though it were a stated pre-tax payroll deduction. FICA applies to the full $92,000 and totals $7,038."
-us,scenario_046,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model wrongly treated the $9,208 employer-sponsored insurance premium as a pre-tax exclusion from Social Security and Medicare wages. With no such exclusion specified, the full $92,000 wage amount generates $7,038 of employee FICA tax."
-us,scenario_046,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so the required value was missing."
-us,scenario_046,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model's $7,193.10 does not result from the employee Social Security and Medicare rates applied to $92,000 and includes an unsupported excess of $155.10. The correct components are $5,704 of Social Security tax and $1,334 of Medicare tax, totaling $7,038."
-us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning calculated the exact $7,038 total but submitted $8,438 as the output value. It failed to carry its computed result into the required numeric field, creating a $1,400 answer inconsistency."
+us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model added $5,704 + $1,334 and got $10,038 instead of $7,038. It then added a made-up $518.40 'Oklahoma state payroll' amount, but Oklahoma has no mandatory employee state payroll tax. The correct total is 7.65% × $92,000 = $7,038."
+us,scenario_046,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model submitted $0 without doing any calculation, even though the head has $92,000 of W-2 wages. Those wages owe 6.2% Social Security ($5,704) plus 1.45% Medicare ($1,334), for $7,038."
+us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $9,208 employer-sponsored insurance premium from FICA wages as a pre-tax cafeteria-plan deduction, but no salary-reduction arrangement is given. The full $92,000 of gross wages is the payroll tax base, so the correct total is 7.65% × $92,000 = $7,038, not $6,333.59."
+us,scenario_046,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model added the $14,839 FLSA overtime premium on top of $92,000, but the prompt says gross wages already include overtime, so overtime was counted twice. It then subtracted the $9,208 ESI premium, which is not a stated pre-tax reduction. FICA applies to $92,000, giving $7,038."
+us,scenario_046,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $9,208 employer-sponsored insurance premium as a Section 125 pre-tax reduction and taxed only $82,792. The household facts give no salary-reduction arrangement, so FICA applies to the full $92,000, giving $5,704 + $1,334 = $7,038."
+us,scenario_046,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model cut FICA wages to $82,792 by subtracting the $9,208 ESI premium as pre-tax, but no cafeteria-plan reduction is given. The payroll tax base is the full $92,000 of gross wages, so the total is $7,038."
+us,scenario_046,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model applied 7.65% to $82,792 after treating the $9,208 ESI premium as a pre-tax deduction from Social Security and Medicare wages. FICA applies to the full $92,000 of gross wages, so the total is $7,038."
+us,scenario_046,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model shrank the FICA base to $82,792 by subtracting the $9,208 insurance premium as pre-tax, which the facts do not support. Social Security of $5,704 and Medicare of $1,334 on the full $92,000 total $7,038."
+us,scenario_046,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value or explanation, so there was nothing to score. The required answer was 7.65% × $92,000 = $7,038."
+us,scenario_046,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model submitted $0 because no state payroll tax rate was specified, which ignores federal FICA altogether. Federal employee Social Security (6.2%) and Medicare (1.45%) apply to the head's $92,000 of wages, giving $7,038."
+us,scenario_046,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"The model's $7,172 is $134 more than 7.65% × $92,000 = $7,038, and it said it was folding in an assumed state payroll tax. Oklahoma has no mandatory employee state payroll tax, so the total is federal Social Security ($5,704) plus Medicare ($1,334) only."
+us,scenario_046,payroll_tax,grok-4.3,llm_error,other,False,"The model used the right base and rates but made an arithmetic slip, answering $7,028. The correct total is 6.2% × $92,000 = $5,704 plus 1.45% × $92,000 = $1,334, which is $7,038."
+us,scenario_046,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model taxed only $82,792 after treating the $9,208 ESI premium as a pre-tax deduction from FICA wages, though no salary-reduction arrangement is given. Applying FICA to the full $92,000 gives $7,038."
+us,scenario_046,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly found no Additional Medicare Tax and no Oklahoma payroll tax. However, it subtracted the $9,208 employer-sponsored premium from wages as pre-tax, which the facts do not support. Social Security and Medicare apply to the full $92,000, giving $7,038."
+us,scenario_046,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value or explanation, so there was nothing to score. The required answer was 7.65% × $92,000 = $7,038."
+us,scenario_046,payroll_tax,minimax-m3,llm_error,other,False,"The model said it applied FICA and Medicare to $92,000 of wages but submitted $7,193.10, which is $155.10 more than the correct 7.65% × $92,000 = $7,038. That points to a wrong combined rate or an arithmetic error, since no Additional Medicare Tax or state payroll tax applies."
+us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly reached $5,704 + $1,334 = $7,038, but it submitted 8438. The final value was mistyped and does not match its own calculation."
us,scenario_046,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_046,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but used the pre-2026 brackets (0.25% to 4.75%) instead of the 2026 schedule (0%/2.5%/3.5%/4.5%, with 4.5% above $14,400). It based the credit on a $4,000 CTC instead of $4,400. It then submitted $3,364, which does not match its own $3,022 result."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and the $220 credit right, but ran the income through the pre-2026 brackets (0.25% to 4.75%) and got $3,221.89. The 2026 joint schedule (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) gives $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It started from federal taxable income using an outdated $14,600 federal standard deduction and applied a nonexistent 5.85% Oklahoma rate. It never used Oklahoma's $12,700 standard deduction, $4,000 of exemptions, or the graduated 2026 schedule, and its final $1,614 does not follow from its own arithmetic."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but applied the pre-2026 brackets with a 4.75% top rate, which gives about $3,221 instead of $2,959.14. It also guessed a credit of about $211 instead of computing 5% of the $4,400 federal CTC ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return with a $17,000 cap, but Oklahoma allows itemizing only when the taxpayer itemizes federally. This household takes the $32,200 federal standard deduction, so the $12,700 Oklahoma standard deduction applies. It also used 4.75% rates and omitted the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It got the $75,300 taxable income right but never worked through the 2026 schedule. The schedule (0%/2.5%/3.5%/4.5%, with 4.5% above $14,400) gives $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its round $2,500 is an estimate based on the old 4.75% rates."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly assumed Oklahoma matches the federal standard deduction and subtracted $30,000 instead of Oklahoma's fixed $12,700. That understated taxable income by $17,300. It also used an invented 3.75% bracket up to $14,400 with 4.75% above, and it ignored the 5%-of-federal-CTC credit."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It first claimed Oklahoma has no income tax, then itemized mortgage interest and property tax on the Oklahoma return. Oklahoma requires federal itemizing for that, and this household takes the federal standard deduction. It then applied pre-2026 4.75% rates instead of taxing $75,303 under the 2026 schedule and subtracting the $220 credit."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used Oklahoma itemized deductions to reach about $49k of taxable income. Because the household takes the $32,200 federal standard deduction, Oklahoma requires the $12,700 standard deduction, which gives $75,303. It also applied the old brackets topping out at 4.75%."
-us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It set the Oklahoma standard deduction equal to the federal one (about $30,900) instead of Oklahoma's $12,700, and it left out the $4,000 of exemptions. It used pre-2026 4.75% brackets and computed the credit as 5% of $4,000 instead of 5% of the $4,400 federal CTC."
-us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut Oklahoma income to $51,795.62 by subtracting the ESI premiums and itemizing, but Oklahoma income starts from the $92,003 AGI. Itemizing is not allowed because the household does not itemize federally, so taxable income is $75,303. It applied the old 4.75% brackets and omitted the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted $26,958.78 of Oklahoma itemized deductions, but Oklahoma allows itemizing only when the taxpayer itemizes federally, and this household takes the $32,200 federal standard deduction. The $12,700 Oklahoma standard deduction should have given $75,303 of taxable income. It also used pre-2026 4.75% brackets and applied no child credit, even though the $220 credit (5% of federal CTC) applies."
-us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It took the $14,839 federal overtime deduction out of AGI and got $77,164. That deduction is taken after AGI is calculated and does not reduce the $92,003 AGI that Oklahoma starts from. It also used old 3.75%/4.75% brackets and omitted the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return ($17,000 capped plus $5,430 medical), but Oklahoma allows itemizing only when the taxpayer itemizes federally. This household takes the federal standard deduction, so the $12,700 standard deduction and $75,303 taxable income apply. It also used 4.75% brackets and computed the credit on one $2,000 CTC ($100) instead of $4,400 ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $3,154 equals the pre-2026 formula ($300 plus 4.75% above $15,200) applied to $75,303, with no child credit. The 2026 schedule gives $2,959.14 of tax, and subtracting the $220 5%-of-CTC credit gives $2,739.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used itemized deductions on the Oklahoma return. Oklahoma requires federal itemizing, and this household takes the $32,200 federal standard deduction, so the $12,700 Oklahoma standard deduction applies. Its $2,105 comes from old 4.75% brackets on about $51.8k of taxable income with no child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut Oklahoma AGI to $82,795 by subtracting the ESI premiums from the reported $92,000 of wages, and it itemized $26,999 even though the household does not itemize federally. It also computed the credit as $100 instead of 5% of the $4,400 federal CTC ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. The correct steps are $75,303 of taxable income taxed at 0%/2.5%/3.5%/4.5% for $2,959.14, minus the $220 credit, for $2,739.14. Its $3,043 overstates the tax by $304, a result that matches using pre-2026 brackets with a 4.75% top rate."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used itemized deductions on the Oklahoma return. Oklahoma requires the $12,700 standard deduction whenever the taxpayer takes the federal standard deduction, as this household does. It also left out the $220 5%-of-federal-CTC credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly used the standard deduction, exemptions, and child credit, but applied the bracket schedule wrongly. The 2026 schedule on $75,303 gives $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its $2,684.51 is $54.63 too low, which comes from misstated lower-bracket amounts."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $12,200 standard deduction instead of Oklahoma's $12,700 joint amount and applied the pre-2026 formula ($300 plus 4.75% above $15,200) instead of the 2026 schedule with 4.5% above $14,400. It based the credit on a $4,000 CTC ($200) instead of $4,400 ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no parseable value for state_income_tax_before_refundable_credits.
-us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the federal overtime deduction out of AGI and used the $32,200 federal standard deduction instead of Oklahoma's $12,700. It dropped the $4,000 of exemptions and applied a flat 4.25% rate instead of the graduated 2026 schedule. It also omitted the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It submitted a $0 placeholder instead of computing the tax. The provided facts fully determine $75,303 of Oklahoma taxable income, $2,959.14 of tax under the 2026 schedule, and $2,739.14 after the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return, which Oklahoma blocks because the household takes the federal standard deduction. It also applied the federal American Opportunity Credit, which has no Oklahoma counterpart. The only Oklahoma nonrefundable credit here is 5% of the $4,400 federal CTC ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right. Its $3,064 regular tax comes from keeping the old lower brackets (0.25% to 3.75% up to $12,200, or $224.50) with 4.5% above $12,200. The 2026 schedule is 0% up to $7,500, 2.5% up to $9,800, 3.5% up to $14,400, and 4.5% above, which gives $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $4,223.29 is more than the tax even the old 4.75% schedule produces on the entire $92,003 AGI (about $4,015). So it did not subtract the $12,700 standard deduction and $4,000 of exemptions to reach $75,303, and it cited an overtime deduction that has no effect on the Oklahoma base."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used a $17,000 capped itemized deduction and got $71,003 of taxable income. Because the household does not itemize federally, the $12,700 Oklahoma standard deduction applies, giving $75,303. It also used the pre-2026 4.75% brackets ($3,018) instead of the 2026 schedule."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a flat 4.5% to about $43,300 of taxable income, which reflects a deduction far larger than Oklahoma's $12,700, and it ignored the 0%/2.5%/3.5% lower brackets. It set the child credit at $100 per child instead of 5% of $2,200 ($110 per child)."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right but misstated the lower brackets. The 0% bracket up to $7,500, 2.5% up to $9,800, and 3.5% up to $14,400 produce $218.50 before the 4.5% bracket, but it counted $240.50, which led to its $2,981.14 figure instead of $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Oklahoma's $12,700 standard deduction and $4,000 of exemptions give $75,303, taxed at $2,959.14, minus the $220 credit, for $2,739.14. Its $2,513 is about $226 too low, which equals roughly $5,000 of extra deductions at the 4.5% rate on top of the $12,700 standard deduction."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right, but its $3,064.14 pre-credit tax keeps the old lower brackets up to $12,200 ($224.50) and applies 4.5% above that. The 2026 schedule (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) gives $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right but understated the lower brackets. It counted $66.50 of tax below the 4.5% bracket instead of $218.50 ($57.50 at 2.5% for $7,500 to $9,800 plus $161 at 3.5% for $9,800 to $14,400), which produced $2,807.14 instead of $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,800 matches taxing about $87,500 under the pre-2026 4.75% schedule, which means it left out the $12,700 Oklahoma standard deduction and did not reach $75,303 of taxable income. It also left out the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted $26,309 of itemized deductions on the Oklahoma return, but Oklahoma allows itemizing only when the taxpayer itemizes federally, and $26,309 is less than the $32,200 federal standard deduction. So the $12,700 Oklahoma standard deduction applies. It also used 4.75% brackets and omitted the $220 5%-of-CTC credit."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized $26,309 on the Oklahoma return even though the household takes the larger $32,200 federal standard deduction. That forces the $12,700 Oklahoma standard deduction and $75,303 of taxable income. It also taxed income above $12,200 at 4.75% instead of using the 2026 schedule, and it left out the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It deducted $26,309 of itemized deductions on the Oklahoma return. Oklahoma requires federal itemizing, which this household does not do, so the $12,700 standard deduction applies. It also wrongly said no Oklahoma nonrefundable credits apply, missing the 5%-of-federal-CTC child credit ($220), and it used the pre-2026 4.75% rate."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return and got $45,216 of taxable income, instead of using the $12,700 standard deduction that applies to a federal non-itemizer, which gives $75,303. It then applied a flat 4.5% with no 0%, 2.5%, or 3.5% lower brackets and no $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It took the federal overtime deduction out of AGI and got $77,164, but that deduction does not reduce the $92,003 AGI that Oklahoma starts from. It also used a $15,000 standard deduction instead of $12,700 and omitted the $220 5%-of-CTC credit."
-us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value for state_income_tax_before_refundable_credits.
-us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly kept overtime in AGI and used the $12,700 standard deduction, but it left out the four $1,000 exemptions, so its $79,303 should have been $75,303. It used the pre-2026 4.75% brackets. It also wrongly said no Oklahoma nonrefundable credits apply, missing the child credit of 5% of the $4,400 federal CTC ($220)."
-us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,773 matches taxing about $87,000 under the pre-2026 4.75% schedule, which means it skipped the $12,700 Oklahoma standard deduction and did not reach $75,303 of taxable income. It also left out the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but kept the old lower brackets up to $12,200 ($224.50) and applied 4.5% above $12,200, instead of the 2026 schedule (0%/2.5%/3.5% up to $14,400). It set the child credit to $0 because the CDCC is $0. The Oklahoma credit is the greater of 20% of the CDCC or 5% of the federal CTC, which is $220 here."
-us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It miscalculated AGI as $94,004 instead of $92,003 and left out the four $1,000 exemptions, which inflated taxable income to $81,304 instead of $75,303. It also used pre-2026 4.75% brackets and omitted the $220 child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It started from a federal-taxable-income figure of $60,503 instead of the $92,003 AGI less Oklahoma's $12,700 standard deduction and $4,000 of exemptions. It applied an invented 0.5% to 5% bracket schedule instead of the 2026 0%/2.5%/3.5%/4.5% schedule, and it left out the $220 5%-of-CTC credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the $75,303 taxable income right but used the pre-2026 schedule (0.25%–4.75%, top rate above $12,200) instead of the 2026 schedule (0% to $7,500, then 2.5%/3.5%, then 4.5% above $14,400). It also based the credit on a $4,000 federal CTC instead of $4,400, giving $200 instead of $220. Its submitted $3,364 doesn't even match its own $3,022 result."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The taxable income ($75,303) and the $220 child credit were both right. The model then used the retired 0.25%–4.75% brackets, which give $3,221.89, instead of the 2026 HB 2764 schedule (0%/2.5%/3.5%/4.5%), which gives $2,959.14 before the credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from federal taxable income with a $14,600 standard deduction instead of federal AGI less Oklahoma's $12,700 standard deduction and $4,000 of exemptions. It applied a flat 5.85% rate that Oklahoma doesn't use, then subtracted an arbitrary amount to reach $1,614 instead of using the 2026 brackets and the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model never computed Oklahoma tax and entered $0 as a placeholder. On $75,303 of taxable income, Oklahoma's 2026 joint brackets produce $2,959.14, and the $220 child credit brings that to $2,739.14."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model computed $75,303 of taxable income correctly but applied the old 0.25%–4.75% brackets (about $3,221) instead of the 2026 schedule with a 0% first bracket and a 4.5% top rate ($2,959.14). Its credit was an imprecise figure of about $211 rather than 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions. Oklahoma requires the $12,700 standard deduction when the filer doesn't itemize federally, and the $32,200 federal standard deduction is larger than this household's itemized total. It also used the old 4.75% top rate and left out the $220 Oklahoma child credit (5% of the federal CTC)."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached about $75,300 of taxable income but never ran the 2026 brackets. Those brackets (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) give $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its rounded $2,500 understates that by about $239."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assumed Oklahoma conforms to the federal standard deduction and subtracted $30,000. Oklahoma's joint standard deduction is fixed at $12,700, so taxable income is $75,303, not $58,003. It also used a made-up bracket schedule with a 4.75% top rate and left out the $220 child credit (5% of the federal CTC)."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model itemized $20,879 on the Oklahoma return. Oklahoma allows itemizing only when the filer itemizes federally, and here the federal standard deduction is larger, so the $12,700 Oklahoma standard deduction applies and taxable income is $75,303. It also used the old 4.75% top rate and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions and got about $49k of taxable income. Because the household takes the standard deduction federally, Oklahoma requires its $12,700 standard deduction, so taxable income is $75,303. It also used the old 4.75% top rate instead of the 2026 4.5% schedule."
+us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model set the Oklahoma standard deduction equal to the federal one (about $30,900), but Oklahoma's joint standard deduction is fixed at $12,700, and it also left out the $4,000 of exemptions. It used the old 4.75% brackets and figured the credit on a $4,000 federal CTC ($200) instead of $4,400 ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used federal itemized deductions to reach $51,795.62 of taxable income. The household takes the standard deduction federally, so Oklahoma's $12,700 standard deduction applies, and with $4,000 of exemptions taxable income is $75,303. It then used the old 4.75% brackets and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,958.78 of Oklahoma itemized deductions instead of the $12,700 standard deduction, which Oklahoma requires because the household doesn't itemize federally. It applied the retired 0.25%–4.75% brackets and said no nonrefundable credit applied, leaving out the 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model cut Oklahoma AGI to $77,164 by subtracting the $14,839 overtime premium. The federal overtime deduction is taken below the line and doesn't reduce the $92,003 federal AGI that Oklahoma starts from. It also used a 4.75% top rate instead of the 2026 4.5% schedule and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions plus $5,430 of medical expenses instead of the $12,700 Oklahoma standard deduction, which applies because the household takes the federal standard deduction. It used the old 4.75% brackets and figured the credit as 5% of $2,000 ($100) instead of 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"There is no reasoning to check, but the correct figure is $2,959.14 in 2026 bracket tax on $75,303, less the $220 child credit. The submitted $3,154 is close to what the retired 0.25%–4.75% schedule gives ($3,221.89) after a partial credit, so the 2026 0%/2.5%/3.5%/4.5% schedule wasn't used."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model said it used itemized deductions, and its $2,105 matches about $51,796 of taxable income under the old 4.75% brackets. Oklahoma requires the $12,700 standard deduction for a filer who doesn't itemize federally, which gives $75,303 of taxable income. The model also left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $9,208 of ESI premiums to get $82,795 of Oklahoma AGI, but federal AGI, which Oklahoma starts from, is $92,003. It also itemized $26,999 instead of taking the $12,700 standard deduction. It used the old 4.75% brackets and took a $100 credit instead of 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"There is no reasoning to check, but the correct figure is $2,959.14 in 2026 bracket tax on $75,303, less the $220 child credit. The submitted $3,043 is close to the $3,064.14 that comes from keeping the old 0.25%–3.75% lower brackets with a 4.5% top rate, which means the model missed the 0% bracket on the first $7,500 and didn't take the full $220 credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied itemized deductions on the Oklahoma return. Because the household takes the larger federal standard deduction, Oklahoma requires its $12,700 standard deduction, and with $4,000 of exemptions taxable income is $75,303. That base yields $2,959.14 before the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the standard deduction, the exemptions and the child credit, but its bracket tax is wrong. On $75,303 the 2026 joint schedule gives $218.50 for the bands up to $14,400 plus 4.5% of $60,903, for $2,959.14, and subtracting $220 leaves $2,739.14. The model's $2,684.51 comes from mis-specified brackets."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $12,200 standard deduction instead of $12,700, giving $75,803 of taxable income instead of $75,303. It applied a 4.75% top rate instead of the 2026 4.5% schedule and figured the credit on a $4,000 federal CTC ($200) instead of $4,400 ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value for state_income_tax_before_refundable_credits, so there was no answer to score against $2,739.14."
+us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model subtracted the federal overtime deduction from AGI to get $77,164. It also applied the $32,200 federal standard deduction instead of Oklahoma's fixed $12,700, skipped the $4,000 of exemptions, used a flat 4.25% rate instead of the graduated 2026 schedule, and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model never computed Oklahoma tax and entered $0 as a placeholder. All the inputs were given: $92,003 of AGI less the $12,700 standard deduction and $4,000 of exemptions gives $75,303, which is $2,959.14 in 2026 bracket tax, or $2,739.14 after the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions instead of the required $12,700 Oklahoma standard deduction. It also counted the federal American Opportunity Credit as a nonrefundable Oklahoma credit, but Oklahoma has no such credit; the only nonrefundable credit here is 5% of the federal CTC, or $220."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model got the $75,303 taxable income and the $220 credit right, but it computed $3,064 by keeping the old 0.25%–3.75% lower brackets up to $12,200 with a 4.5% top rate. The 2026 schedule taxes the first $7,500 at 0%, the next $2,300 at 2.5% and $9,800–$14,400 at 3.5%, which gives $2,959.14."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The submitted $4,223.29 is more than Oklahoma rates would charge on the entire $92,003 of AGI, so the model didn't apply the $12,700 standard deduction and $4,000 of exemptions to the 2026 schedule. The federal overtime deduction it mentioned doesn't affect Oklahoma income either. The correct computation is $75,303 of taxable income, $2,959.14 of tax, less the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions. Oklahoma requires the $12,700 standard deduction because the household takes the larger federal standard deduction, so taxable income is $75,303, not $71,003. Its $3,018 of bracket tax also comes from the old 4.75% schedule rather than the 2026 4.5% schedule."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied a flat 4.5% rate to a base of about $43,303 instead of running the graduated 2026 schedule on $75,303, which means it subtracted a much larger standard deduction than Oklahoma's $12,700. It also used $100 per child ($200 total) instead of 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model got the taxable income and the $220 credit right but computed $2,981.14 of bracket tax instead of $2,959.14. The bands up to $14,400 come to $218.50 (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400), and the model's figure is $22 too high, so it mis-set the lower-bracket thresholds or rates."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $2,513 is $226 below the correct $2,739.14. That gap fits taxing only the 4.5% top band ($2,740.64) and dropping the $218.50 owed in the 2.5% and 3.5% brackets before subtracting the credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct $220 credit to $3,064.14 of bracket tax, which comes from keeping the old 0.25%–3.75% lower brackets up to $12,200 with a 4.5% top rate. The 2026 HB 2764 schedule has a 0% first bracket to $7,500, 2.5% and 3.5% bands to $14,400 and 4.5% above, which gives $2,959.14."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model applied 4.5% above $14,400 ($2,740.64) correctly, but it computed only $66.50 on the first $14,400. The correct amount is $218.50: 2.5% on $7,500–$9,800 plus 3.5% on $9,800–$14,400. That understated the pre-credit tax by $152."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's $3,800 is higher than the retired 4.75% schedule gives on the correct $75,303 ($3,221.89), so it overstated both the base and the rates. It also left out the $220 Oklahoma child credit, which is 5% of the federal CTC."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,309 of itemized deductions instead of the $12,700 Oklahoma standard deduction, which applies because the household takes the larger federal standard deduction. It then used the old 4.75% brackets and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model itemized $26,309 on the Oklahoma return. Oklahoma requires the $12,700 standard deduction when the filer doesn't itemize federally, so taxable income is $75,303, not $61,694. It also used the old 4.75% rate above $12,200 and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,309 of itemized deductions instead of the $12,700 standard deduction, which Oklahoma requires because the household takes the federal standard deduction. It used the old 4.75% brackets and said no Oklahoma nonrefundable credits applied, leaving out the $220 credit (5% of the $4,400 federal CTC)."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied a flat 4.5% rate to $45,216 of income after itemized deductions. Oklahoma taxable income is $75,303, after the $12,700 standard deduction and $4,000 of exemptions, and it is taxed on the graduated 2026 schedule (0%/2.5%/3.5%/4.5%), less the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model started from $77,164 of AGI by subtracting the federal overtime deduction, which is taken below the line and leaves AGI at $92,003. It used a standard deduction of about $15,000 instead of Oklahoma's $12,700 and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value for state_income_tax_before_refundable_credits, so there was no answer to score against $2,739.14."
+us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model left out the four $1,000 Oklahoma exemptions and got $79,303 of taxable income instead of $75,303. It used the retired 4.75% brackets and said no nonrefundable credit applied, missing the Oklahoma child credit of 5% of the $4,400 federal CTC ($220)."
+us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"There is no reasoning to check, but the model's $3,773 is above even what the retired 4.75% schedule gives on the correct $75,303 base ($3,221.89). The correct computation applies the 2026 0%/2.5%/3.5%/4.5% brackets for $2,959.14 and subtracts the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the 4.5% top rate but kept the old 0.25%–3.75% lower brackets up to $12,200, which gives $3,064.14 instead of $2,959.14 under the 2026 schedule. It also checked only the Oklahoma EITC and child care credit and missed that the Oklahoma child care/child tax credit is the greater of 20% of the federal CDCC or 5% of the federal CTC, which here is $220."
+us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model misadded AGI as $94,004 when it is $92,003 and left out the $4,000 of exemptions, which produced $81,304 of taxable income instead of $75,303. It used the old brackets and left out the $220 child credit."
+us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed federal taxable income ($60,503). Oklahoma starts from federal AGI and subtracts its own $12,700 standard deduction and $4,000 of exemptions, which gives $75,303. It also made up a 0.5%–5% bracket schedule and left out the $220 child credit."
us,scenario_046,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_046,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age and assets below $2,000 as sufficient for Ohio aged Medicaid while never establishing that the head passed the applicable income test. The head's wages and Social Security total $34,626, SSI is zero, and no Medicaid category applies."
@@ -4126,7 +4201,8 @@ us,scenario_048,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli
us,scenario_048,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the head's income ""very low"" and inferred Medicaid eligibility directly from age and residence. The head's MAGI income is 2.17 times FPL, SSI is zero, and age 85 alone does not supply a qualifying Medicaid pathway."
us,scenario_048,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted eligibility based on age and income without identifying or applying any Ohio Medicaid pathway. The head's age does not independently confer eligibility, and the income facts result in no qualifying Medicaid category."
us,scenario_048,head_medicare_eligible,ox-alpha,llm_error,other,False,"The model correctly identified the age-based Medicare eligibility pathway and explicitly concluded that the head is eligible, but then encoded that conclusion as value = 0. Under the required binary contract, Medicare eligible is 1, so the numeric submission contradicted its own reasoning."
-us,scenario_048,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct payroll-tax base and components but miscalculated their amounts, rounded the annual liability to a whole dollar, and then submitted $782 despite its own stated total of $586. Applying 6.2% and 1.45% to $7,656 yields $474.67 and $111.01, respectively, for $585.68."
+us,scenario_048,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model correctly applied the 7.65% employee FICA rate to $7,656 of wages and got $585.68. Its explanation ends with 'value = 586', but the numeric value field holds 575. That $575 does not follow from any payroll-tax computation; it is a transcription error between the explanation and the submitted value."
+us,scenario_048,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model applied 6.2% Social Security plus 1.45% Medicare to $7,656 and reached about $586, with small arithmetic slips ($474.55 instead of $474.67, and $111.04 instead of $111.01). It then submitted 782 anyway. It hinted at an 'additional mandatory state payroll tax' that does not exist in Ohio, and the 782 contradicts its own stated total of 586."
us,scenario_048,snap,claude-opus-4.7,llm_error,thresholds_rates,False,The model correctly calculated that the benefit formula produced $0 but then applied the $23 minimum allotment to an income-ineligible household. It also submitted $23 as an annual amount after stating that $23 was monthly and that twelve months would equal $276.
us,scenario_048,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model explicitly found that net income exceeded its stated eligibility limit and that the household was ineligible, yet submitted $2,376. It applied a positive allotment after its own eligibility calculation had already required a $0 benefit."
us,scenario_048,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model treated age 85 as bypassing the controlling income screen and assumed the minimum allotment remained available. The household’s $34,626 income exceeds the $17,420 eligibility threshold, so no minimum benefit is payable."
@@ -4134,52 +4210,49 @@ us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals
us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0."
us,scenario_048,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate to the first dollar of taxable income, ignoring Ohio's zero-tax bracket: Ohio's rate schedule imposes no tax on Ohio taxable nonbusiness income at or below the roughly $26,050 threshold, and this filer's $7,656 of wages (Social Security survivor benefits are excluded from federal AGI here because provisional income of $21,141 is under the $25,000 base amount) falls far below it, yielding $0. It compounded the error by inventing a nonexistent $1,400 Ohio 'single standard deduction' — Ohio starts from federal AGI and allows personal exemptions, not a standard deduction — and then taxed the residual $3,856 at 2.75% to reach $106."
us,scenario_048,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model taxed the entire $7,656 of Ohio taxable income at 2.75% ($210.54), skipping Ohio's zero-rate first bracket that exempts all Ohio taxable nonbusiness income up to about $26,050, under which this household's liability is $0 before any credit is considered. It also mis-specified the personal exemption as a credit equal to 2.75% of $2,400 ($66) — Ohio subtracts the $2,400 exemption from Ohio adjusted gross income and separately allows only a $20-per-exemption credit — and then netted a $50 senior citizen credit against a tax that never existed, landing on $94.54."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own computation reached $30,830, but it submitted $33,223, a figure that does not follow from that work. The $30,830 also leaves out the $580 OBBBA non-itemizer cash charitable deduction and the $721 above-the-line traditional IRA deduction, so its taxable income of $190,185 is $1,301 above the frozen reference's; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), leaving only the $580 missing."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its work matches the reference except for the $721 traditional IRA contribution, which it disallowed under the §219(g) active-participant phaseout on the grounds that the spouse's 401(k) deferrals make them a plan participant. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. That $721 explains the entire $158.63 gap to the frozen reference (22% × $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used an outdated $29,200 standard deduction instead of the 2026 figure of $32,200 and never took the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA overtime premium. Its tax of $48,837 is also far above what 2026 joint brackets produce even on its own $217,364 of taxable income (about $37k)."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $25,000 overtime deduction, used a $31,500 standard deduction instead of $32,200, and added the Additional Medicare Tax, which is a payroll tax, to income tax. It then submitted $49,861, which bears no relation to the roughly $36.6k it had just computed."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It used a stale $30,300 standard deduction and 2025-style brackets and never applied the $25,000 qualified overtime deduction. It then submitted $53,503 even though its own calculation came to $37,055."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It used the $40,000 SALT cap as the deduction amount instead of the actual $20,655 of real estate taxes, which pushed itemized deductions to $46,081 and wrongly made itemizing beat the $32,200 standard deduction. It also left out the $25,000 overtime deduction, and $47,500 is far more than the brackets produce on the roughly $201k of taxable income it stated."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly disallowed the $721 traditional IRA contribution because the spouse is covered by a workplace plan. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. Everything else matches, so the $158.63 gap to the frozen reference comes from the $721 (22% of $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 values, a $30,000 standard deduction and 2025 bracket thresholds, instead of the 2026 figures ($32,200 standard deduction, 22% bracket up to $211,400). It also never applied the OBBBA qualified overtime deduction, which is $25,000 on the spouse's $44,647 FLSA premium, so taxable income came out at $217,385 instead of $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and used a $32,600 standard deduction. It then raised its own $38,831 estimate to $62,700 by pointing to Additional Medicare Tax and phase-outs, neither of which belongs in income tax before credits."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the full $928 of auto loan interest from AGI, a deduction the reference does not allow for this household. It also left out the $580 non-itemizer cash charitable deduction and the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Against the frozen reference net deductions were $373 too low and tax $82 too high; against the corrected value, $30,702.59, net deductions were $348 too high and tax $76.59 too low."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction and used a $31,000 standard deduction instead of $32,200. That left taxable income at $216,385 instead of $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law (pre-TCJA brackets of 10/15/25/28% and personal exemptions), but OBBBA made the TCJA rate schedule and standard deduction permanent. It also subtracted the $16,408 of employer-paid premiums from wages that do not include them and never took the $25,000 overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its stated method removes the $16,408 of employer-sponsored premiums from wages, but those premiums were never part of the $253,000. Its $36,639 corresponds to taxable income of about $216k under 2026 joint brackets, roughly $27.7k above the reference, which fits with leaving out the $25,000 overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It correctly used the $32,200 standard deduction and the 2026 brackets but never took the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium. It also left out the $580 non-itemizer charitable deduction and the $721 IRA deduction, so taxable income was $215,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA provisions sunset in 2026 and so used personal exemptions, uncapped itemizing and 10/15/25% brackets, but OBBBA made the TCJA structure permanent and added the $25,000 overtime deduction, which it never applied. It also subtracted $16,408 of employer-paid premiums from wages that exclude them."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It stated taxable income of about $229,000, roughly $40k above the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), because it did not take the $25,000 overtime deduction on top of the $32,200 standard deduction. Even so, $52,367 is higher than 2026 joint brackets produce on $229k."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 rules (personal exemptions, uncapped itemized deductions and pre-TCJA brackets) instead of OBBBA's permanent TCJA structure, and it never took the $25,000 overtime deduction. It also cut wages by $16,408 of employer-paid premiums that are not in the $253,000 of gross wages."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-TCJA-expiration schedules, with $10,100 of personal exemptions, itemized deductions and a 25% bracket, instead of OBBBA's permanent $32,200 standard deduction and 10/12/22% brackets. It also left out the $25,000 qualified overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $42,203 corresponds to taxable income of about $240k under 2026 joint brackets, meaning it applied neither the $25,000 overtime deduction nor most of the $32,200 standard deduction against AGI of roughly $247k."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,408 of employer-paid premiums from wages that never included them and used a $30,000 standard deduction instead of $32,200. It also never took the $25,000 qualified overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It worked from gross wages, interest and dividends with only generic allowable deductions. Its $37,921 corresponds to taxable income of about $220k, which is what AGI yields without the $25,000 qualified overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2025 figures, a $30,000 standard deduction and 2025 brackets, instead of the 2026 figures of $32,200 and a 22% bracket up to $211,400. It also never applied the $25,000 qualified overtime deduction, so taxable income was $217,385 instead of $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction but never took the $25,000 qualified overtime deduction, leaving taxable income at $215,185. It then overstated ordinary tax on $208,977 as $36,982; the 2026 brackets give $35,399."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It built its deductions around itemizing with the old $10,000 SALT cap, which never beats the $32,200 standard deduction here. Its $20,460 corresponds to taxable income of only about $143k, some $46k below the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), so its deduction stack was badly overstated."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $38,566 corresponds to taxable income of about $219k under 2026 joint brackets, which fits with leaving out the $25,000 qualified overtime deduction that brings taxable income down to $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,408 of employer-sponsored insurance premiums from wages. Those premiums are employer-paid and were never part of the $253,000 of gross wages, so AGI came out at $230,977. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income is $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the $928 of auto loan interest as a qualified vehicle-loan-interest deduction, which the reference does not allow for this household, and it left out the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Taxable income ended up $207 below the frozen reference's, producing the $45.53 shortfall; against the corrected value, $30,702.59, the $928 deduction alone leaves it $204.21 short."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,408 of employer-paid health premiums as a pretax reduction to wages that never included them, which cut taxable income to $173,197. That is $15,687 below the frozen reference's $188,884. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so the whole gap to the corrected value comes from the $16,408."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $33,473 corresponds to taxable income of about $202k, some $13k above the frozen reference's $188,884, so it did not apply the full stack of $32,200 standard, $25,000 overtime and $580 charitable deductions (the frozen reference also deducts the $721 IRA contribution, which the exclusion's corrected value disallows under the active-participant phase-out, 26 U.S.C. 219(g)). It also claimed a vehicle-loan-interest deduction that the reference does not allow."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $16,408 of employer-sponsored premiums even though the $253,000 of gross wages never included them, giving AGI of $230,977. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income was $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It stopped at the standard deduction plus the overtime deduction. It never took the OBBBA non-itemizer cash charitable deduction of $580 (available from 2026) or the $721 above-the-line IRA deduction, so taxable income was $1,301 above the frozen reference's. The frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so against the corrected value, $30,702.59, only the $580 is missing and its $30,830.14 is $127.55 high."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its joint AGI of $247,385 leaves out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That difference alone produces the $158.63 gap to the frozen reference, and its $30,702.54 is $0.05 below the corrected value."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It excluded $16,408 of employer-plan premiums from wages that never contained them, which put AGI at $230,977 instead of $246,664, the frozen reference's AGI net of a $721 IRA deduction. The model left out that deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income came out $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It said AGI was about $263k, ignoring the 401(k) exclusion. It itemized about $20k under caps, which is below the $32,200 standard deduction, and never took the $25,000 overtime deduction, so taxable income came out around $230k instead of $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (uncapped itemizing, $10,500 of personal exemptions and pre-TCJA brackets) even though OBBBA made the TCJA schedule permanent. It also left out the $25,000 qualified overtime deduction, leaving taxable income at $210,149."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed 2026 income under post-TCJA-sunset 10/15/25% brackets with personal exemptions of about $10,910 instead of OBBBA's permanent $32,200 standard deduction and 10/12/22% brackets. It also omitted the $25,000 overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $580 non-itemizer cash charitable deduction and the $721 IRA deduction, so taxable income was $190,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so of the two omissions only the $580 is an error. It also overstated ordinary tax on $183,977 as $30,051; the 2026 brackets give $29,899."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It built taxable income from SALT-capped itemized deductions instead of the larger $32,200 standard deduction and never applied the $25,000 qualified overtime deduction. Its $36,370 corresponds to taxable income of about $215k."
-us,scenario_049,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $230,256 comes from subtracting the $16,408 of employer-paid premiums, which are not in wages, and it then took only the $32,200 standard deduction. It never applied the $25,000 overtime deduction or the $580 non-itemizer charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA provisions had expired and used itemized deductions, two personal exemptions and pre-TCJA brackets, but OBBBA made the TCJA schedule and the $32,200 standard deduction permanent for 2026. It also subtracted $16,408 of employer premiums from wages and never took the $25,000 overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It started from $247,385, leaving out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That is the only difference, and it produces the $158.63 gap to the frozen reference; its $30,702.54 is $0.05 below the corrected value."
-us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It treated the overtime deduction as an above-the-line adjustment of $25,675, above the $25,000 joint cap, and used a $30,900 standard deduction instead of $32,200. It also ran all $190,089 of taxable income, qualified dividends included, through the ordinary brackets and then added 15% on the $6,208 of dividends again, taxing them twice."
-us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $721 IRA contribution, as the frozen reference does although the exclusion's corrected value disallows it under the active-participant phase-out (26 U.S.C. 219(g)), and used the $32,200 standard deduction, but it never applied the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium or the $580 non-itemizer charitable deduction. Taxable income came out at $214,464 instead of $188,884."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It miscomputed AGI as $258,821; $262,821 minus $15,436 is $247,385. It then used $15,395 of itemized deductions even though that is less than the standard deduction, and it never took the $25,000 overtime deduction. It finally submitted $51,214.75, which is unrelated to its own $42,933."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $18,160 of retirement contributions, which includes the after-tax $2,724 Roth 401(k), and never took the $25,000 qualified overtime deduction. It then reported $44,587 of ordinary tax on $205,441, far above the roughly $34.8k that the 2026 joint brackets give."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own derivation reached $30,830. That figure omits the 2026 non-itemizer cash charitable deduction (~$580, worth about $128 of tax) but is otherwise correct. It then submitted $33,223, a number unrelated to its arithmetic."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and the non-itemizer charitable deduction. It also deducted the $721 traditional IRA, which is fully phased out for an active plan participant at this income, and used a stale $29,200 standard deduction instead of $32,200. The $48,837 it reported is not supported by any bracket calculation; correct 2026 brackets on its own $217k taxable income give about $36.6k."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It reported $0 tax on $247,385 of AGI. The correct path is $189,605 of taxable income after the $32,200 standard deduction, $25,000 overtime deduction and ~$580 charitable deduction, which yields $30,703 of tax with no nonrefundable credits to offset it."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction, and used a $31,500 standard deduction instead of $32,200. It also wrongly added the 0.9% Additional Medicare Tax to income tax. It then submitted $49,861, which does not match its own $36,571 calculation."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction, and used a $30,300 standard deduction with outdated bracket thresholds. Even so, its own work produced $37,055, yet it submitted an unrelated $53,503."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the $40,000 SALT cap as the deductible amount, but only the $20,655 of real estate taxes actually paid is deductible. That inflated itemized deductions to $46,081 when they are under $27k, below the $32,200 standard deduction. It also missed the $25,000 qualified overtime deduction, which is available whether or not the household itemizes."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction ($25,000 joint cap, no phase-out at $247k MAGI) and the non-itemizer cash charitable deduction. It also used the 2025 standard deduction of $30,000 and 2025 bracket thresholds instead of 2026's $32,200 and $24,800/$100,800/$211,400."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 overtime deduction and the charitable deduction, and deducted the $721 IRA contribution even though that deduction is fully phased out. Its own estimate was about $38,831, but it then inflated the answer to $62,700 by adding Medicare and phase-out effects that do not belong in income tax."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $928 of auto-loan interest. That deduction drops by $200 per $1,000 of MAGI over $200,000, so at $247,385 it is fully eliminated. It also missed the ~$580 non-itemizer cash charitable deduction; the net $348 of extra deductions explains the $77 shortfall."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction or the ~$580 non-itemizer charitable deduction, and used a $31,000 standard deduction instead of $32,200. That overstated taxable income by more than $26k."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expired in 2026 and used pre-TCJA 15/25/28% brackets, personal exemptions and uncapped itemizing. OBBBA made the TCJA rates and the $32,200 standard deduction permanent and added the $25,000 overtime deduction, which it omitted. It also subtracted the $16,408 employer-sponsored insurance premium from wages, although only the traditional 401(k) reduces the stated wages."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted the employer-sponsored insurance premium from wages and did not apply the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. Its $36,639 implies taxable income of roughly $215k against the correct $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, standard deduction and 2026 brackets were correct, but it omitted the $25,000 qualified overtime deduction and the $579.76 non-itemizer charitable deduction. That gap is exactly 22% × $25,579.76 = $5,627.55, the full amount of its overstatement."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expired and used pre-TCJA brackets, personal exemptions and uncapped itemized deductions. Under OBBBA the 2026 rules are the $32,200 standard deduction, TCJA-style brackets and the $25,000 overtime deduction, which it omitted. It also subtracted the $16,408 employer-sponsored insurance premium from the stated wages."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used about $229,000 of taxable income, which means it skipped the $25,000 qualified overtime deduction and the full $32,200 standard deduction. Correct taxable income is $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-expired law (personal exemptions, uncapped real estate taxes, pre-TCJA brackets), but OBBBA kept the TCJA structure for 2026. It omitted the $25,000 qualified overtime deduction and wrongly subtracted the employer-sponsored insurance premium from wages."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used TCJA-expired rules (personal exemptions of $10,100, 25% bracket, itemizing $26,736) instead of 2026 OBBBA law, which has a $32,200 standard deduction and TCJA brackets. It never applied the $25,000 qualified overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $42,203 implies taxable income far above the correct $189,605, consistent with skipping the $25,000 qualified overtime deduction and the non-itemizer charitable deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from the stated gross wages, but only the $15,436 traditional 401(k) reduces them. It also used a $30,000 standard deduction and omitted the $25,000 qualified overtime deduction and the ~$580 charitable deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reported $37,921 with no overtime deduction mentioned. That amount reflects taxable income above $215k, so it left out the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction that bring taxable income to $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets instead of 2026's $32,200 and the $100,800 top of the 12% bracket. It also omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer cash charitable deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was nothing to score."
+us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction but never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. It also miscomputed ordinary tax: $36,982 instead of $35,399 on $208,977 under 2026 brackets."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It produced $20,460 with no computation. That understates the tax on $189,605 of taxable income, which is $29,771 of ordinary tax plus $931 on qualified dividends."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $38,566 reflects taxable income about $35k above the correct $189,605, consistent with omitting the $25,000 qualified overtime deduction and using a smaller standard deduction or older brackets."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from wages, which cut AGI to $230,977. Only the $15,436 traditional 401(k) reduces the stated gross wages, so AGI is $247,385. The extra exclusion understated taxable income by $16,408, or $3,610 of tax at 22%."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It took the $928 qualified vehicle-loan interest deduction. That deduction drops by $200 per $1,000 of MAGI over $200,000, so at $247,385 it is zero. The extra $928 of deductions accounts for the $204 shortfall."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 employer-sponsored insurance premium as a pre-tax reduction of the stated gross wages. Only the $15,436 traditional 401(k) is excluded, so AGI is $247,385, not $230,977. That understated taxable income by $16,408."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $33,473 implies taxable income of about $202,200, so it allowed only about $13,000 of overtime and vehicle-loan deductions. The correct deductions are the full $25,000 overtime deduction plus the ~$580 non-itemizer charitable deduction, and the vehicle-loan deduction is fully phased out above $200k MAGI."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from wages, giving AGI of $230,977 instead of $247,385. Only the traditional 401(k) reduces the stated gross wages, and the extra exclusion understated tax by about $3,610."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the 2026 brackets, the standard deduction and the $25,000 overtime deduction correctly, but omitted the 2026 non-itemizer cash charitable deduction ($579.76). That accounts exactly for the $127.55 overstatement (22% × $579.76)."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It excluded the $16,408 employer-sponsored insurance premium from wages, although only the $15,436 traditional 401(k) reduces the stated gross wages. That understated AGI and taxable income by $16,408 and tax by about $3,610."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used AGI of about $263k (no 401(k) exclusion), took about $20k of itemized deductions instead of the $32,200 standard deduction, and omitted the $25,000 overtime deduction. That produced taxable income of about $230k against the correct $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA-expired 2026 law: personal exemptions of $10,500, uncapped property taxes and pre-TCJA brackets. Under OBBBA the TCJA brackets and the $32,200 standard deduction continue, and the $25,000 qualified overtime deduction, which it omitted, is added."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-expired rules: 10/15/25% brackets, about $10,910 of personal exemptions and itemizing $26,736. 2026 OBBBA law uses the $32,200 standard deduction and TCJA brackets, plus the $25,000 qualified overtime deduction, which it never took."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the ~$580 non-itemizer cash charitable deduction, so taxable income came out at $190,185 instead of $189,605. It also miscomputed ordinary tax on $183,977 as $30,051; the 2026 brackets give $29,899."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It itemized with a capped SALT deduction and never applied the $25,000 qualified overtime deduction or the $32,200 standard deduction. The $36,370 it reported implies taxable income of about $214k against the correct $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI of $230,256 subtracts both the $16,408 employer-sponsored insurance premium and the phased-out $721 IRA contribution, neither of which reduces AGI. It also omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used TCJA-expired rules (personal exemptions, pre-TCJA brackets, itemizing) and subtracted the employer-sponsored insurance premium from wages. It omitted the $25,000 qualified overtime deduction and the $32,200 standard deduction that apply under 2026 OBBBA law."
+us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It ran its full $190,089 taxable income, including the $6,208 of qualified dividends, through the ordinary brackets and then taxed those dividends again at 15%, counting them twice. It also used a made-up $25,675 overtime cap (the correct deduction is $25,000), deducted the phased-out $721 IRA, and used a $30,900 standard deduction instead of $32,200."
+us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction. It also deducted the $721 traditional IRA, which is fully phased out for an active plan participant at $247k MAGI."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It made three deduction errors. It miscomputed AGI as $258,821 ($262,821 − $15,436 = $247,385), chose $15,395 of itemized deductions over the larger $32,200 standard deduction, and omitted the $25,000 overtime deduction. It then submitted $51,215, which does not match its own $42,933 figure."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It deducted the phased-out $721 IRA plus Roth contributions ($18,160 total), when only the $15,436 traditional 401(k) is excludable. It used a $32,900 standard deduction and omitted the $25,000 overtime deduction. It also overcomputed ordinary tax: $44,587 on $205,441, when 2026 brackets give about $34.4k."
us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -4188,37 +4261,38 @@ us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou
us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's reasoning correctly determined that the 53-year-old head lacks both the age threshold and any qualifying early-Medicare condition, but it submitted value 1 instead of 0. Its numeric output directly contradicts its own eligibility analysis and required final marker."
us,scenario_049,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right: $11,439 Social Security on the $184,500 base, $3,668.50 Medicare and $27 Additional Medicare Tax, for $15,134.50. It then submitted an unrelated $14,503.98 that none of its own arithmetic supports."
-us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. It applied the $200,000 single threshold instead of the $250,000 married-filing-jointly threshold, which inflated Additional Medicare Tax to $477. It also added a made-up $904.13 New Hampshire payroll tax. Its listed components add to $15,502.83, not the $20,502.83 it submitted."
-us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 wage base of $176,100, giving $10,918.20, instead of the 2026 base of $184,500, which gives $11,439. The result is $520.80 too low. Its Medicare and Additional Medicare figures were correct."
-us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning twice arrived at the correct $15,134.50: $11,439 + $3,668.50 + $27. It then submitted $13,164.85, a number that contradicts its own computation."
-us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It briefly used an estimate of $183,600, then dropped it and kept the 2025 wage base of $176,100 for Social Security, giving $10,918.20. The 2026 base of $184,500 gives $11,439. That understated the total by $520.80."
-us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It labeled $176,100, the 2025 figure, as the 2026 Social Security wage base and got $10,918.20 of Social Security tax. The 2026 base of $184,500 gives $11,439. Its Medicare and Additional Medicare figures were correct."
-us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up 2026 wage base of $180,900, giving $11,215.80, instead of the actual $184,500, which gives $11,439. That understated Social Security tax by $223.20."
-us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,408 employer-sponsored insurance premiums from the $253,000 gross wages, which cut Medicare tax to $3,430.58. The lower wages also fell below the $250,000 threshold, so it dropped the $27 Additional Medicare Tax. It also used the 2025 wage base of $176,100 instead of $184,500."
-us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"$14,850.98 equals 6.2% of a $184,200 wage base ($11,420.40) plus 1.45% of $236,592 ($3,430.58). That means it removed the $16,408 insurance premiums from Medicare wages and so omitted the $27 Additional Medicare Tax. It also used a Social Security base slightly below the correct $184,500."
-us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated $236,592, which is gross wages minus the $16,408 premiums, as Medicare wages. That undercounted Medicare tax by $237.92 and wrongly concluded there was no Additional Medicare Tax. It also used a guessed wage base of $183,900 instead of $184,500."
-us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 base of $176,100 instead of the 2026 base of $184,500. Even its own method gives $14,613.70, not the $13,959.81 it submitted, so the final number also contains an arithmetic error."
-us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It reduced FICA wages to $236,592 by excluding the $16,408 insurance premiums. That lowered Medicare tax to $3,430.58 and removed the $27 Additional Medicare Tax. Its $14,814 total also implies a wage base of about $183,600 rather than $184,500."
-us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It computed Medicare tax on $236,592 after subtracting the insurance premiums and left out the $27 Additional Medicare Tax. It also used a projected wage base of $174,900, which gives $10,844, instead of $184,500, which gives $11,439."
-us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"$4,058.45 is about the size of the Medicare tax alone. The model left out nearly all of the $11,439 Social Security tax owed on the first $184,500 of the spouse's wages. The correct components are $11,439, $3,668.50 and $27."
-us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get $236,592 of FICA wages, so it understated Medicare tax and dropped the $27 Additional Medicare Tax. It also capped Social Security at a guessed $181,200 instead of the 2026 base of $184,500."
-us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"$14,613.34 matches a calculation using the 2025 wage base of $176,100, which gives $10,918.20 of Social Security tax. The 2026 base of $184,500 gives $11,439, so the answer is short by about $521."
-us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It explicitly used a $176,100 wage base, which is the 2025 figure, to get $10,918.20 of Social Security tax. The 2026 base is $184,500, which gives $11,439."
-us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"$19,648 is about $4,500 above the correct total. That gap is consistent with applying the 6.2% Social Security rate to wages above the $184,500 cap instead of stopping at $11,439, since uncapped Social Security on all $253,000 is $15,686."
-us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave $15,302 with no component breakdown, $167.50 above the correct total. The correct figures are $11,439 of Social Security on the $184,500 base, $3,668.50 of Medicare on $253,000, and $27 of Additional Medicare Tax on the $3,000 above the $250,000 married-filing-jointly threshold. The model overstated at least one of these."
-us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It excluded the $16,408 insurance premiums from FICA wages, which gave Medicare tax of $3,430.58 and no Additional Medicare Tax. It also used a wage base of $183,600 instead of the 2026 base of $184,500."
-us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Its Social Security figure of $11,439 was correct. However, it computed Medicare on wages reduced by the $16,408 insurance premiums, $3,430.58 instead of $3,668.50. That also put wages under $250,000, so it wrongly dropped the $27 Additional Medicare Tax."
-us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used the correct $184,500 base for Social Security. It then took the $16,408 insurance premiums off Medicare wages, leaving $236,592. That understated Medicare tax by $237.92 and wrongly removed the $27 Additional Medicare Tax on the $3,000 of gross wages above $250,000."
-us,scenario_049,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Its $11,439 of Social Security tax was correct. It treated the $16,408 insurance premiums as pre-tax and cut Medicare wages to $236,592. That lost $237.92 of Medicare tax and the $27 Additional Medicare Tax that applies to $253,000 of gross wages."
-us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a rough wage base of about $180,000 and then rounded the whole total to $14,000. The exact figures are $184,500 × 6.2% = $11,439, plus $3,668.50 and $27."
-us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It capped Social Security at a guessed 2026 wage base of $183,600, giving $11,383. The actual base is $184,500, which gives $11,439. Its Medicare and Additional Medicare figures were correct."
-us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applied, but the spouse's $253,000 of wages is $3,000 above the $250,000 married-filing-jointly threshold, which adds $27. Its $14,808 total is consistent with Medicare computed on premium-reduced wages and a Social Security base below $184,500."
-us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get FICA wages of $236,592. That gave Medicare tax of $3,430.58 instead of $3,668.50 and removed the $27 Additional Medicare Tax. Its $11,439 of Social Security tax was correct."
-us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It reduced both Social Security and Medicare wages by the $16,408 insurance premiums, so it understated Medicare tax and dropped the $27 Additional Medicare Tax. It also used a $183,000 wage base, giving $11,346, instead of $184,500, giving $11,439."
-us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The $19,354.50 it submitted equals 6.2% of the full $253,000 ($15,686) plus $3,668.50 of Medicare. That means it applied Social Security tax with no wage-base cap. None of the capped totals in its own reasoning used the correct 2026 base of $184,500 either."
-us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"Its components are $11,475 of Social Security, which is slightly above the correct $11,439, plus $3,668.50 and $27. They add to $15,170.50, but it reported $23,780.75, so the addition itself is wrong."
+us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning gets the right total: $11,439 Social Security on the $184,500 base, $3,668.50 Medicare and $27 Additional Medicare Tax, for $15,134.50. It then submitted an unrelated $14,503.98, so the output does not match its own calculation."
+us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. It applied the $200,000 single threshold for Additional Medicare Tax instead of the $250,000 joint threshold ($477 instead of $27). It also invented $904.13 of New Hampshire payroll tax, though New Hampshire has none. Its listed parts add to $15,502.83, not the $20,502.83 it submitted."
+us,scenario_049,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It said Social Security, Medicare and Additional Medicare Tax apply to the spouse's $253,000 in wages but submitted 0. It left out the $11,439 + $3,668.50 + $27 = $15,134.50 it should have reported."
+us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That understates Social Security tax by $520.80 ($10,918.20 instead of $11,439); its Medicare and Additional Medicare Tax figures were correct."
+us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning gets the right total: $11,439 + $3,668.50 + $27 = $15,134.50, using the $184,500 base. It then submitted $13,164.85, which does not match its own calculation."
+us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It briefly tried an estimated 2026 base of $183,600, then dropped it and used the 2025 base of $176,100. The 2026 base is $184,500, so Social Security tax should be $11,439, not $10,918.20, which left its total $520.80 short."
+us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It treated $176,100 (the 2025 base) as the 2026 Social Security wage base. The 2026 base is $184,500, so it understated Social Security tax by $520.80. Its Medicare and Additional Medicare Tax figures were correct."
+us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up 2026 wage base of $180,900 instead of $184,500. That gives $11,215.80 of Social Security tax instead of $11,439; the rest of its figures were correct."
+us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,408 employer-sponsored insurance premiums from FICA wages. That cut Medicare tax to $3,430.58 and wrongly removed the $27 Additional Medicare Tax, which applies on all $253,000. It also used the 2025 base of $176,100 instead of $184,500."
+us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Its $14,850.984 equals 1.45% of $236,592 (wages minus the $16,408 premiums) plus 6.2% of $184,200. So it excluded the premiums from Medicare wages, dropping the $27 Additional Medicare Tax, and used a wage base $300 below the 2026 base of $184,500."
+us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get $236,592 of Medicare wages. That understated Medicare tax and wrongly removed the $27 Additional Medicare Tax. It also used a guessed wage base of $183,900 instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2025 base of $176,100 instead of the 2026 base of $184,500. Its submitted $13,959.81 is even below the $14,613.70 its own method produces, so it also made an arithmetic error on top of the stale base."
+us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut FICA wages to $236,592 by subtracting the $16,408 premiums. That lowered Medicare tax to $3,430.58 and wrongly removed the $27 Additional Medicare Tax. Its $14,814 also implies a $183,600 wage base instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It used $236,592 of Medicare wages, after subtracting the $16,408 premiums, instead of the full $253,000, and so wrongly dropped the $27 Additional Medicare Tax. It also used a guessed 2026 base of $174,900 instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"Its $4,058.45 is about Medicare tax alone, so it left out nearly all of the $11,439 employee Social Security tax on the spouse's wages, which run past the $184,500 base."
+us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums to get $236,592 of FICA wages. That cut Medicare tax and wrongly removed the $27 Additional Medicare Tax on wages above $250,000. It also used a $181,200 base instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $14,613.34 matches 6.2% of the 2025 base of $176,100 plus Medicare and Additional Medicare Tax. It needed the 2026 base of $184,500, which gives $11,439 of Social Security tax."
+us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used $176,100 (the 2025 base) as the Social Security cap instead of the 2026 base of $184,500. That understated Social Security tax by $520.80; its other parts were correct."
+us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no payroll_tax value or explanation, so there was no answer to score against $15,134.50."
+us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It named the correct parts but submitted $19,648, about $4,500 too high. A figure that high only results if Social Security tax is applied above the $184,500 cap; the capped amount is $11,439."
+us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $15,302 is $167.50 too high. That fits a Social Security base above the 2026 cap of $184,500. The correct parts are $11,439 Social Security, $3,668.50 Medicare and $27 Additional Medicare Tax."
+us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums from FICA wages. That lowered Medicare wages to $236,592 and set Additional Medicare Tax to $0 instead of $27. It also used a $183,600 base instead of $184,500."
+us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Its Social Security figure ($11,439) is right. It subtracted the $16,408 premiums from Medicare wages, so Medicare tax came out at $3,430.58 instead of $3,668.50, and it skipped the $27 Additional Medicare Tax due on wages above $250,000."
+us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used the correct $184,500 base but subtracted the $16,408 premiums to get $236,592 of Medicare wages. That understated Medicare tax by $237.92 and wrongly put the household under the $250,000 Additional Medicare Tax threshold, losing $27."
+us,scenario_049,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $16,408 premiums as reducing payroll-taxable wages to $236,592. Medicare tax and Additional Medicare Tax both apply to the full $253,000, so it lost $237.92 of Medicare tax and the $27 surtax."
+us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a rough Social Security base of about $180,000 instead of $184,500 and rounded the total down to $14,000. The exact sum is $11,439 + $3,668.50 + $27 = $15,134.50."
+us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 Social Security wage base at $183,600 instead of $184,500. That cost $55.80 of Social Security tax, and it also rounded Medicare tax to $3,669."
+us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applies, which only holds if the $16,408 premiums are subtracted from Medicare wages. Its $14,808 matches 1.45% of $236,592 plus Social Security on a base of about $183,600, instead of the full $253,000 and the $184,500 base."
+us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums to get $236,592 of FICA wages. That gave $3,430.58 of Medicare tax instead of $3,668.50 and wrongly removed the $27 Additional Medicare Tax; its $11,439 Social Security figure was right."
+us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums from Social Security and Medicare wages. That dropped Medicare wages below the $250,000 threshold and removed the $27 surtax. It also used a $183,000 base instead of $184,500."
+us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"Its submitted $19,354.50 is 7.65% of the full $253,000, so it ignored the Social Security wage base cap altogether. Its reasoning also wavered between the 2024 and 2025 bases ($168,600 and $176,100) instead of the 2026 base of $184,500."
+us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"Its parts ($11,475 + $3,668.50 + $27) add to $15,170.50, but it submitted $23,780.75, a plain addition error. Its $11,475 Social Security figure is also not 6.2% of the 2026 base of $184,500, which is $11,439."
us,scenario_049,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -4239,44 +4313,45 @@ us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,thr
us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It itemized the interest and dividends correctly, subtracted the $4,800 joint exemption to reach $5,021, and then multiplied by a 1% rate it believed applied in 2026. That rate comes from the 2021 phase-down schedule, which the 2023 law replaced. The tax was repealed effective 2025, so the correct 2026 amount is $0, not $50.21."
us,scenario_049,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached $2,641.60, which was already $21.60 too high because it taxed the $180 capital gain distribution at 12% instead of 0%. It then submitted $3,086, a number with no derivation in its reasoning."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $12,400 bracket, but it labeled the $180 of non-Schedule D capital gains as short-term and taxed it at 12%. Capital gain distributions are long-term gains taxed at 0% here, so the tax on $23,900 of ordinary income is $2,620, not $2,641.60."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It subtracted the $5,389 employer-sponsored premium from gross wages and used 2025-era figures ($15,000 deduction, $11,600 bracket), reaching about $2,103. It then zeroed out the tax by pointing to the EITC, which is refundable and by definition excluded from tax before refundable credits."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It first computed $2,642 with the correct 2026 parameters, while wrongly taxing the $180 capital gain distribution at 12%. It then switched to invented figures ($16,000 deduction, $12,150 bracket) and submitted $2,960, which those figures do not produce either."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed $2,642 from the correct $16,100 deduction and $12,400 bracket, with the $180 capital gain distribution wrongly taxed at 12% instead of 0%. It then submitted $2,867 with no arithmetic behind the change."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It set up the right 2026 computation ($1,240 plus $1,402 = $2,642), with the $180 capital gain distribution wrongly taxed at 12%. It then submitted $2,955, which is not the sum of its own bracket amounts."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on 2025 parameters (a $15,000 standard deduction and a $11,925 10% bracket) instead of the 2026 values of $16,100 and $12,400, which overstated taxable income at $25,180. It also explicitly taxed the $180 of non-Schedule D gains as ordinary income rather than at the 0% capital gains rate."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used the correct $16,100 deduction with a $12,150 bracket and reached $2,646.60. It then added about $400 by double-counting the $180 of capital gains 'at ordinary rates', even though they were already in taxable income and are taxed at 0% as capital gain distributions."
-us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $11,925 10% bracket instead of the 2026 values of $16,100 and $12,400. Although it said the capital gains were taxed at 0%, its arithmetic put the full $24,780, including the $180, at ordinary rates."
-us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,493 pre-TCJA standard deduction, a $4,779 personal exemption and a 15% bracket. The TCJA structure was made permanent, so the correct figures are a $16,100 deduction, no exemption and 10%/12% rates. It also wrongly subtracted the $5,389 employer premium from gross wages."
-us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a post-TCJA-expiration schedule ($8,600 standard deduction, $4,950 personal exemption, 15% second bracket) instead of the permanent 2026 rules. Those rules give a $16,100 deduction, no exemption and a 12% second bracket, for $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 of gains at 0%, but it used a $15,400 standard deduction and a $12,200 bracket instead of the 2026 values of $16,100 and $12,400. That left $24,600 of ordinary taxable income instead of $23,900."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,300 standard deduction, a $5,150 personal exemption and a 15% bracket, and it subtracted the $5,389 employer premium from wages. Those offsetting errors land near $2,620 by coincidence. The correct computation is $40,000 minus the $16,100 deduction, taxed at 10%/12%."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used a stale $15,000 standard deduction and then subtracted a $1,300 nonrefundable Child Tax Credit, even though the filer has no children and no nonrefundable credit applies."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $5,389 employer premium from gross wages and applied a pre-TCJA-style deduction plus personal exemption of about $13,400. The correct figures are the 2026 $16,100 standard deduction against the full $40,000 of wages, with no exemption."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It treated 2026 as a TCJA-expiration year, using an $8,300 deduction, a $5,150 exemption and a 15% rate on income above $11,600, and it subtracted the $5,389 employer premium from wages. Under the permanent 2026 rules, the $16,100 deduction and 10%/12% brackets give $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $40,000 minus $16,100 = $23,900 taxable, which gives $2,620. Its $2,388 matches about $21,970 of taxable income, so it over-deducted roughly $1,930."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer premium from gross wages, even though listed wages are the gross amount. It also used a $15,700 deduction and the 2025 $11,925 bracket, which understated taxable income at $19,091 instead of $23,900."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied an expired-TCJA schedule with a standard deduction, a personal exemption and a 15% second bracket. Under the permanent 2026 law, the $16,100 deduction and 12% second bracket give $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 of gains at 0%, but it assumed the TCJA expired. It used an $8,300 deduction plus a $5,300 exemption and a 15% rate instead of the 2026 $16,100 deduction and 12% second bracket."
-us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used the 2024 $14,600 deduction and the 2025 $11,925 bracket and computed $2,831.10. It then submitted $2,530 with a premium-tax-credit discussion that supplies no arithmetic. The PTC has no role in this measure."
-us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It ended the 10% bracket at $12,200 instead of $12,400, and it taxed the $180 of non-Schedule D gains as ordinary income at 12% instead of at the 0% capital gains rate."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out all tax. The 2026 $16,100 deduction leaves $23,900 of taxable wage income, which is taxed at 10%/12% for $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave only a vague reference to 'nonrefundable taxes reduced'. The correct computation is $23,900 taxable, which gives $2,620. Its $2,146 matches taxable income about $3,950 too low, as if an extra deduction had been taken."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated a $15,350 standard deduction and a $12,000 bracket instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 capital gain distribution at 12% instead of 0%."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the correct $16,100 deduction and 2026 brackets, but it taxed the $180 of non-Schedule D capital gains at the 12% ordinary rate. Those capital gain distributions are long-term gains taxed at 0%, which drops the tax by $21.60 to $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied the correct 2026 deduction and brackets, but it put the $180 capital gain distribution in the 12% bracket. That $180 is taxed at the 0% long-term capital gains rate, so the tax is $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It included the $180 of non-Schedule D capital gains in ordinary taxable income at 12%. As capital gain distributions they are taxed at 0% in this bracket, so only the $23,900 of wage taxable income is taxed, giving $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,500, which gave $24,680 of taxable income, instead of the 2026 $16,100. It also taxed the $180 capital gain distribution at ordinary rates instead of 0%."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025 parameters (a $15,000 deduction and a $11,925 bracket) rather than the 2026 values of $16,100 and $12,400. It also taxed the $180 capital gain distribution at 12% instead of 0%."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the inflation-adjusted 2026 $16,100. It also taxed the $180 capital gain distribution at 12% instead of the 0% rate."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $15,400 deduction and a $12,225 bracket instead of the 2026 values of $16,100 and $12,400. It also treated the $180 of non-Schedule D gains as ordinary income instead of taxing it at 0%."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted pre-TCJA rules' ($8,509 deduction, $5,427 exemption, 15% bracket) and subtracted the $5,389 employer premium from gross wages. Under the permanent 2026 law, $40,000 minus the $16,100 deduction, taxed at 10%/12%, gives $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there was no answer to score."
-us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It used the correct $16,100 deduction but a $12,750 bracket. It then invented a roughly $545 'nonrefundable credit for net capital gains', when the 0% capital gains rate only removes tax on the $180 of gains, which is worth $21.60."
-us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly applied the $16,100 deduction and the $12,400 10% bracket, but it taxed the $180 of non-Schedule D capital gains at 12%. Those capital gain distributions are taxed at 0%, which gives $2,620 instead of $2,641.60."
-us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $2,618.60 using a $16,300 deduction and a $12,350 bracket in place of the 2026 values of $16,100 and $12,400. It then submitted $1,410, a number that has no support in its own arithmetic."
-us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It claimed the standard deduction and nonrefundable credits cut the tax to zero. The childless filer has no nonrefundable credits, and the $16,100 deduction leaves $23,900 taxable, which gives $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own arithmetic gave $2,641.60, which already taxed the $180 capital gain distribution at 12% instead of 0%, and it then submitted an unrelated $3,086 that none of its steps supports. Correct: $23,900 of ordinary taxable income taxed at 10%/12% = $2,620, plus $0 on the $180."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It ran the $180 of non-Schedule-D capital gain distributions through the ordinary 12% bracket. Those gains get the preferential 0% rate because taxable income is below the 0% LTCG threshold. That overstated tax by $21.60 ($2,641.60 instead of $2,620)."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and used the outdated $15,000 deduction and $11,600 bracket. It then set the tax to $0 by citing the EITC, but the EITC is refundable and excluded from this measure. A childless 25-year-old earning $40,000 is also far above the EITC phase-out. The correct pre-refundable tax is $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It had the right $16,100 standard deduction but never ran the brackets, rounding to $2,700 instead of computing 10% × $12,400 + 12% × $11,500 = $2,620 on the $23,900 of ordinary income. It also treated the $180 capital gain as ordinary income instead of taxing it at 0%."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It computed $2,642, which taxed the $180 capital gain distribution at 12% instead of 0%, and then switched to $2,960 using a $16,000 deduction and a $12,150 bracket. Those figures produce about $2,670, not $2,960, so the final number has no basis in its own work. The correct 2026 parameters ($16,100 deduction, $12,400 bracket) give $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It reached $2,642 with the correct 2026 deduction and brackets, then submitted $2,867 with no supporting calculation. It also taxed the $180 capital gain distribution at 12% instead of the 0% preferential rate, which gives $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It worked out roughly $2,642 with the correct $16,100 deduction and $12,400 bracket, then submitted $2,955 without any arithmetic to support it. Taxing the $180 capital gain distribution at 0% gives $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and $11,925 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also included the $180 capital gain in ordinary income at 12% rather than 0%. Together these overstate tax at $2,783 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a $12,150 10%-bracket ceiling instead of $12,400 and got $2,646.60. It then inflated that to $3,059 by citing 'adjustments' that do not exist. The $180 capital gain distribution is taxed at 0%, not at ordinary rates, so the tax is $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and the 2025 $11,925 bracket ceiling instead of the 2026 values of $16,100 and $12,400. Although it said the capital gains were taxed at 0%, it ran the full $24,780 (including the $180) through ordinary rates, producing $2,735 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, applying a personal exemption and a 15% bracket. The OBBBA made the TCJA rates permanent with a $16,100 standard deduction and no exemption. It also wrongly subtracted the $5,389 employer insurance premium from the stated $40,000 of wages."
+us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law (an $8,600 standard deduction, a $4,950 personal exemption and a 15% second bracket) instead of the permanent post-OBBBA structure: a $16,100 deduction, no exemption and 10%/12% brackets. It also taxed the $180 gain at ordinary rates. The result is $3,385.75 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0%, but it used a $15,400 standard deduction and a $12,200 bracket ceiling instead of the 2026 values of $16,100 and $12,400. That left ordinary income at $24,600 instead of $23,900 and tax at $2,708 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,300 deduction, a $5,150 exemption and a 15% bracket, and it subtracted the $5,389 employer premium from wages that were already stated as $40,000. Its $2,621.15 lands near $2,620 only because these errors offset each other; the correct path is $40,000 − $16,100 = $23,900 at 10%/12%."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used the outdated $15,000 standard deduction and then subtracted a $1,300 Child Tax Credit, but this childless single filer qualifies for no nonrefundable credits. The correct tax is $2,620, with $0 in credits."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and applied a pre-TCJA standard deduction plus personal exemption of about $13,400. Under 2026 law the only deduction is the $16,100 standard deduction, leaving $23,900 taxed at 10%/12% = $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,300 deduction, a $5,150 exemption and a 15% bracket, and it subtracted the $5,389 employer premium from the stated wages. Its $2,621.15 is only coincidentally close to the correct $2,620, which comes from $23,900 of ordinary income at 10%/12%."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Under 2026 brackets its $2,388 corresponds to about $21,970 of ordinary taxable income, roughly $1,930 below the correct $23,900 ($40,000 − $16,100), which at 10%/12% yields $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and used a $15,700 deduction and the 2025 $11,925 bracket ceiling. That cut taxable income to $19,091 and tax to $2,052.42. The correct figures are a $16,100 deduction and $23,900 of ordinary income, giving $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a pre-TCJA personal exemption and a 15% second bracket for 2026. The OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no exemption and a 12% second bracket. That overstates tax at $3,392 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0%, but it assumed the TCJA expired, using an $8,300 deduction, a $5,300 exemption and a 15% bracket. Under current 2026 law ($16,100 deduction, 10%/12% brackets), the $23,900 of ordinary income is taxed at $2,620, not $3,350."
+us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used the 2024 $14,600 deduction and the 2025 $11,925 bracket and got $2,831.10. It then submitted $2,530 after an irrelevant discussion of the premium tax credit, a refundable credit that cannot affect this measure. The correct 2026 calculation gives $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $12,200 10%-bracket ceiling instead of the 2026 value of $12,400, and it taxed the $180 capital gain distribution at the ordinary 12% rate instead of 0%. That produces $2,645.60 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out all tax, but the $16,100 deduction against $40,000 of wages leaves $23,900 of taxable income. At the 10%/12% brackets that is $2,620 of tax, with no offsetting nonrefundable credits."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no real derivation. Under 2026 brackets its $2,146 corresponds to about $19,950 of taxable income, roughly $3,950 below the correct $23,900 ($40,000 − $16,100), which yields $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,350 and the 10% ceiling at $12,000; the actual values are $16,100 and $12,400. It also taxed the $180 gain at 12% rather than 0%, overstating tax at $2,830.60 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It ran the $180 of non-Schedule-D capital gain distributions through the ordinary brackets. Those distributions are taxed at the 0% preferential rate at this income level, so its answer is $21.60 too high ($2,641.60 instead of $2,620)."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It included the $180 capital gain distribution in the ordinary 12% bracket. That gain is taxed at the 0% preferential rate because taxable income is below the 0% LTCG threshold, so tax is $2,620, not $2,641.60."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It taxed all $24,080 at ordinary 10%/12% rates. The $180 non-Schedule-D capital gain distribution gets the 0% preferential rate, leaving only $23,900 taxed at ordinary rates, for $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $24,680 of taxable income implies a standard deduction of about $15,500 instead of the 2026 value of $16,100. Its $2,781 does not match any 2026 bracket computation on that income. The correct result is $23,900 of ordinary income taxed at $2,620, with $0 on the $180 gain."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and $11,925 bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 gain at 12% instead of 0%, giving $2,783 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 post-OBBBA $15,750 standard deduction instead of the 2026 value of $16,100, and it taxed the $180 capital gain distribution at 12% rather than 0%. That gives $2,684 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,225 bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 gain at ordinary rates instead of 0%, producing $2,729.10 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed a reversion to pre-TCJA rules, with an $8,509 deduction, a $5,427 exemption and a 15% bracket. It also subtracted the $5,389 employer premium from the stated $40,000 of wages. Current 2026 law is a $16,100 deduction with 10%/12% brackets, giving $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used inflated bracket ceilings (10% up to $12,750) and then subtracted a fictitious ~$545 'nonrefundable credit for net capital gains'. The 0% rate on the $180 gain only removes the $21.60 it would otherwise owe on that gain, leaving $2,620, not $2,090."
+us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied the ordinary 12% rate to the $180 of non-Schedule-D capital gain distributions instead of the 0% preferential rate. That overstates tax by $21.60: $2,641.60 instead of $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $2,618.60 using a slightly wrong $16,300 deduction and $12,350 bracket, then submitted $1,410, which contradicts its own arithmetic and corresponds to no credit or deduction. The correct figure is $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits reduce the tax to zero, but a childless single filer with no child-care, education or other qualifying expenses has no nonrefundable credits. The liability on $23,900 of ordinary taxable income is $2,620."
us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model awarded $560 of childless EITC without applying the phaseout endpoint for a single filer. At $40,000 of earnings, the childless EITC has fully phased out."
us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract."
us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own EITC calculation correctly reached $0 and stated that no other refundable credits apply, but it submitted $2,557.60. The submitted value contradicts its completed derivation and does not represent any applicable refundable-credit component."
@@ -4294,166 +4369,170 @@ us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract.
us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation."
us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which produced $830.40. It then submitted $1,020, which contradicts its own arithmetic. The correct result is 3% × ($40,180 − $12,835) = $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 single standard deduction. It missed that Act 11 indexes the deduction to inflation starting in 2026, which gives $12,835. That overstated taxable income by $335 and tax by $10.05."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model claimed Louisiana has no individual income tax. Louisiana taxes individual income at a flat 3% of federal AGI minus the $12,835 standard deduction, which yields $820.35 here."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model got the flat 3% structure right but used the base-year $12,500 standard deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed $830.40 using the unindexed $12,500 deduction. It then subtracted a nonexistent 'personal credit' to reach $632, but Louisiana's post-reform law has no personal exemption credit for this filer. The correct deduction is $12,835 and the tax is $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 deduction and then made an unexplained 'rounding' cut from about $830 to $812. The correct figure is 3% of ($40,180 − $12,835) = $820.35, using the 2026 inflation-indexed deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 standard deduction instead of the 2026 indexed $12,835. Taxable income came out $335 too high, giving $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model deducted about $2,783 of federal income tax, a deduction Louisiana repealed after 2021. It also used the unindexed $12,500 standard deduction instead of $12,835. Together these understated taxable income and produced $747 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model computed $830.40 with the unindexed $12,500 deduction. It then submitted $1,080, citing an exemption and credits that would lower tax, not raise it. The correct figure is 3% × $27,345 = $820.35, using the $12,835 indexed deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model assumed the $12,500 standard deduction carries into 2026 unchanged. The 2026 inflation-indexed deduction is $12,835, which gives $820.35 instead of $830.40."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model paired the new 3% flat rate with the repealed pre-2025 $4,500 exemption and counted that amount twice, as a standard deduction and a personal exemption. It should have subtracted the single $12,835 standard deduction. That overstated taxable income at $31,180 instead of $27,345."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a nonexistent 4.25% flat rate instead of 3%. It reduced AGI to $34,791 by subtracting ESI premiums that are not in the AGI computation, and used a $4,500 exemption instead of the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the 2025 $12,500 single standard deduction. It missed the 2026 inflation adjustment to $12,835, so it overstated tax by $10.05 at $830.40."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model subtracted the unindexed $12,500 standard deduction instead of the 2026 indexed $12,835. That produced $830.40 instead of 3% × $27,345 = $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the repealed pre-2025 law: 1.85%/3.5% brackets, a $4,500 exemption, and a federal income tax deduction. It also cut AGI to $34,791 by subtracting ESI premiums. It should have used the flat 3% rate on $40,180 minus the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said it applied Louisiana's 'progressive' rates, but Louisiana has had a single 3% rate since 2025. The correct figure is 3% × ($40,180 − $12,835) = $820.35. The $724 answer reflects the obsolete graduated schedule, not the flat-tax computation."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used the repealed 1.85%/3.5% brackets and $4,500 exemption instead of the flat 3% rate with the $12,835 standard deduction. It also started from $34,791 instead of the $40,180 federal AGI, having wrongly subtracted ESI premiums."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied pre-2025 law: the federal income tax deduction, a $4,500 exemption, and 1.85%/3.5% brackets. It also started from a $34,791 AGI net of ESI premiums. For 2026 it should have used the flat 3% rate on $40,180 minus the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $1,115 exceeds the correct $820.35 (3% × ($40,180 − $12,835)), which fits using a far smaller deduction than $12,835 or the obsolete graduated rates."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the repealed 1.85%/3.5% brackets with a $4,500 exemption instead of the flat 3% rate and $12,835 standard deduction. It also started from $34,791 instead of the $40,180 AGI, having wrongly subtracted ESI premiums."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the pre-2025 1.85%/3.5% brackets and $4,500 exemption to $40,180. It should have used the 2025-onward flat 3% rate and the 2026 $12,835 standard deduction, so it overstated taxable income and tax."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the obsolete $4,500 deduction and 1.85%/3.5% brackets instead of the flat 3% rate on AGI minus the $12,835 standard deduction. That produced $1,042.55 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction for 2026 without Act 11's inflation indexing, which raises it to $12,835. That overstated taxable income by $335 and tax by $10.05."
-us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835. That yielded $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model claimed deductions fully offset Louisiana tax. The only deduction is $12,835, which leaves $27,345 taxable at 3% and produces $820.35 of tax."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave no derivation. Its $781 does not match the flat 3% on $40,180 minus the 2026 $12,835 standard deduction, which yields $820.35, so it did not apply the actual 2026 deduction and rate."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. That gave $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 deduction instead of the 2026 indexed $12,835. Taxable income came out $335 too high and tax $10.05 too high."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied the base-year $12,500 single deduction and missed the 2026 inflation indexing to $12,835. That produced $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated Louisiana taxable income at $27,680 instead of $27,345."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 single standard deduction for 2026 instead of $12,835. That yielded $830.40 instead of 3% × $27,345 = $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model assumed a $12,500 standard deduction for 2026 and missed that it is inflation-indexed to $12,835. That overstated tax by $10.05."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model subtracted the unindexed $12,500 standard deduction instead of the 2026 $12,835 amount. That left $27,680 taxable instead of $27,345 and $830.40 of tax instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no computation. The correct figure is the flat 3% on $40,180 minus the $12,835 2026 standard deduction, which yields $820.35. The $680 answer understates that, which fits using an oversized deduction or the obsolete graduated rates."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model subtracted the federal $15,000 standard deduction instead of Louisiana's own $12,835 deduction. It then applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction instead of the 2026 inflation-indexed $12,835, then rounded $830.40 to $830. The correct result is $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the base-year $12,500 single deduction and missed the 2026 inflation adjustment to $12,835. That overstated tax at $830.40."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model stacked repealed pre-2025 provisions: a federal income tax deduction, separate $4,500 standard deduction and personal exemption, and 1.85%/3.5% brackets. It also started from a $34,791 AGI net of ESI premiums. It should have used the flat 3% on $40,180 minus $12,835."
-us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 $12,835 and rounded $830.40 to $830. The correct result is $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for this output, so there is nothing to score. The correct figure is 3% × ($40,180 − $12,835) = $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 standard deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used the obsolete $4,500 deduction on wages alone, omitting the $180 capital gain, and computed $1,065. It then submitted an unrelated $1,208. The correct computation is 3% of $40,180 minus the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the unindexed $12,500 single deduction instead of the 2026 $12,835. Taxable income came out $335 too high and tax $10.05 too high."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model subtracted the federal $15,750 standard deduction plus a repealed $4,500 exemption and applied invented 1.75%/3.55% brackets. It then submitted $573, which contradicts its own $482.52. It should have used the flat 3% on $40,180 minus Louisiana's $12,835 deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave no deduction or rate detail. Its $1,324.80 far exceeds 3% × ($40,180 − $12,835) = $820.35, so it applied a wrong rate or omitted most of the $12,835 standard deduction."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the inflation-adjusted 2026 amount of $12,835, and calculated $830.40. It then submitted $1,020, which does not follow from its own calculation. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"It applied the 2025 single standard deduction of $12,500 without the 2026 inflation indexing that Act 11 requires, which raises it to $12,835. Taxable income was overstated by $335, so the tax came out at $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"It claimed Louisiana has no individual income tax. Louisiana taxes individual income at a flat 3% on AGI minus the $12,835 single standard deduction, which comes to $820.35 here."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"It claimed the standard deduction and personal exemption wipe out all liability. The 2026 Louisiana standard deduction is $12,835 and there is no separate personal exemption, so $27,345 of income is taxed at 3%, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"It cited Act 11 correctly but used the base-year $12,500 single standard deduction and missed the 2026 inflation indexing to $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"It calculated $830.40 with the unindexed $12,500 deduction and then subtracted a made-up 'personal credit' to get $632. The reform eliminated the personal exemption and its credit, and the correct figure is 3% × ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"It started from the unindexed $12,500 standard deduction and then cut the result to $812 with a made-up 'rounding of federal AGI adjustments'. No such adjustment exists. The correct calculation uses the indexed $12,835 deduction and gives $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835, so taxable income came out at $27,680 instead of $27,345 and the tax at $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"It subtracted a federal income tax deduction of $2,783, but Louisiana's 2024 reform repealed that deduction starting in 2025. It also used the unindexed $12,500 standard deduction instead of $12,835, and arrived at $747 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"It calculated $830.40 using the unindexed $12,500 deduction and then submitted $1,080 with no supporting step. The correct calculation uses the 2026 indexed deduction of $12,835 and gives 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"It assumed a $12,500 standard deduction and missed the 2026 inflation adjustment to $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"It combined the 3% flat rate with the pre-reform $4,500 standard deduction and $4,500 personal exemption, which overstated taxable income at $31,180. Under the reform the only deduction is the single standard deduction, $12,835 for 2026, so taxable income is $27,345 and the tax is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"It applied a 4.25% rate that does not exist in Louisiana law; the 2026 rate is a flat 3%. It also reduced AGI to $34,791 by subtracting the ESI premiums and used the old $4,500 exemption. The correct calculation is 3% × ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835, which overstated tax by $10.05 at $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 indexed standard deduction of $12,835, which gave $27,680 of taxable income and $830.40 of tax instead of $27,345 and $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"It used pre-2025 Louisiana law, with the 1.85%/3.5% brackets, the federal income tax deduction and the $4,500 combined exemption, all of which Act 11 replaced for 2025 and later. It also reduced AGI to $34,791 by subtracting ESI premiums instead of using $40,180. The correct calculation is a flat 3% on $40,180 − $12,835, which is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"It applied Louisiana's 'progressive' rate schedule, but the graduated brackets were repealed for 2025 and later and replaced by a flat 3% rate. The correct calculation is 3% × ($40,180 − $12,835) = $820.35, not $724."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"It used the repealed 1.85%/3.5% brackets and the old $4,500 exemption, and started from an AGI of $34,791 that wrongly removes the ESI premiums. Under the 2026 rules, the $40,180 AGI minus the $12,835 standard deduction is taxed at a flat 3%, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted a federal income tax deduction and a $4,500 exemption and applied the 1.85%/3.5% brackets, all of which the 2024 reform eliminated starting in 2025. Under the 2026 rules, a flat 3% applies to $40,180 minus the $12,835 standard deduction, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At the 3% rate, $1,115 implies about $37,167 of taxable income, which means it effectively left out most of the $12,835 standard deduction. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 combined deduction and the 1.85%/3.5% brackets, applied to an AGI wrongly reduced to $34,791 by the ESI premiums. Under the 2026 rules, a flat 3% applies to $40,180 − $12,835, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"It applied the repealed $4,500 standard deduction/exemption and the 1.85%/3.5% graduated brackets. Act 11 replaced these for 2025 and later with a flat 3% rate and a single standard deduction that is $12,835 in 2026, which gives $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-2025 $4,500 deduction and the 1.85%/3.5% rate schedule instead of the reform's flat 3% rate with a $12,835 standard deduction for 2026, which overstated the tax at $1,042.55 against $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 base amount of $12,500 for the single standard deduction and did not apply the 2026 inflation indexing to $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"It applied the $12,500 standard deduction without the 2026 inflation adjustment to $12,835, which overstated taxable income by $335 and the tax by $10.05."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"It claimed deductions and exemptions eliminate all Louisiana tax. The $12,835 standard deduction leaves $27,345 taxable at 3%, which gives $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"It gave no actual calculation. At 3%, $781 implies about $26,033 of taxable income, which means a deduction of roughly $14,147 instead of the $12,835 single standard deduction. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"It subtracted the 2025 single standard deduction of $12,500 instead of the 2026 indexed $12,835, so it taxed $27,680 instead of $27,345 and got $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 standard deduction without the 2026 inflation adjustment to $12,835, which overstated the tax at $830.40 against $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"It applied the base-year $12,500 single standard deduction instead of the inflation-indexed 2026 amount of $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction and missed that Act 11 indexes it for inflation, to $12,835 in 2026, so the tax came out at $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"It assumed a $12,500 standard deduction instead of the 2026 indexed $12,835, which overstated taxable income by $335 and the tax by $10.05."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 single deduction instead of the 2026 amount of $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"It applied the 2025 base standard deduction of $12,500 instead of the 2026 inflation-adjusted $12,835, so it taxed $27,680 instead of $27,345 at 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $680 implies about $22,667 of taxable income, which means roughly $17,500 of deductions instead of the $12,835 single standard deduction. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,000 standard deduction instead of Louisiana's own 2026 single standard deduction of $12,835. It then applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate, and got $675 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 amount of $12,835 and rounded $830.40 to $830. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 inflation-indexed single standard deduction of $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"It used pre-reform law throughout: a federal income tax deduction, a $4,500 standard deduction plus a $4,500 exemption, and the 1.85%/3.5% brackets. It also cut AGI to $34,791 by subtracting the ESI premiums. Under the 2026 rules, a flat 3% applies to $40,180 − $12,835, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835 and rounded the result to $830. The correct liability is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there was no answer to score. The correct calculation is 3% × ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without the 2026 inflation indexing to $12,835, which gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 standard deduction instead of the 2026 single standard deduction of $12,835 and left out the $180 of capital gains. It then submitted $1,208, which does not match its own $1,065 calculation. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"It subtracted the 2025 base deduction of $12,500 instead of the 2026 inflation-indexed $12,835, which overstated the tax by $10.05 at $830.40."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,750 standard deduction plus a $4,500 Louisiana exemption and applied made-up 1.75%/3.55% brackets. It then submitted $573, which does not match its own $482.52 calculation. Louisiana in 2026 taxes $40,180 minus its own $12,835 standard deduction at a flat 3%, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $1,324.80 implies $44,160 of taxable income, which is more than the $40,180 AGI, so it either used a rate above 3% or took no standard deduction. The correct figure is 3% × ($40,180 − $12,835) = $820.35."
us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own work produced about $107,277, using a $32,600 standard deduction instead of $32,200 and ordinary tax of about $106,932 instead of about $103,866 on the 2026 brackets. It then submitted $117,285, a figure about $10,000 above its own calculation with no derivation behind it."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: 2026 brackets, the $32,200 standard deduction, $127.20 on qualified dividends and $218.42 NIIT. The one difference is that it disallowed the $43 traditional IRA contribution under the active-participant phase-out, as the exclusion's corrected value does under 26 U.S.C. 219(g), while the frozen reference subtracts $43.28 above the line (AGI $504,778.56) because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That extra $43 of income taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It reached the frozen reference's AGI ($504,779), which takes the $43 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), but used a $28,700 standard deduction instead of the 2026 OBBBA $32,200 and never ran the 2026 brackets. Its final $103,241 is an unsupported figure. The frozen reference's path is ordinary tax of $103,865.78 on $471,730.56, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, without the IRA deduction, is $104,225.26."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax, which is a payroll tax and not part of income tax before refundable credits. It also used outdated bracket thresholds (24% ending at $400,000 instead of $403,550) and a $32,300 standard deduction. It then inflated its own $107,040 to $110,997 with no calculation to support the increase."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its guessed bracket thresholds (24% ending at $401,950 instead of $403,550, with lower thresholds throughout) overstated ordinary tax by about $378. It also left the $926 401(k) deferral in AGI, used a $32,300 standard deduction, and omitted the $218.42 NIIT on $5,748 of net investment income."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its bracket schedule applied to about $471,350 of ordinary income works out to roughly $104,000, but it reported $141,500, an arithmetic breakdown of about $37,000. It also left out the $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax (Form 8959) to income tax, although it is a payroll tax outside this output. It also used a $30,000 standard deduction instead of the 2026 OBBBA $32,200 and its own inflated bracket guesses instead of the 2026 schedule."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the annual $926 401(k) and $43 IRA contributions as weekly amounts, multiplied them by 52 and capped the 401(k) at $31,000, which understated AGI by about $32,000. It then replaced its own $94,754 with an arbitrary $121,500. It also used 2025 brackets and left out NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: the $32,200 standard deduction, 2026 brackets, $127 on qualified dividends and $218 NIIT. The one difference is that it treated the $43 traditional IRA contribution as phased out, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That $43 taxed at 32% is the $14 gap to the frozen reference, and its $104,225 is $0.26 below the corrected value, $104,225.26."
-us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It reported ordinary tax of $119,769 on about $474k of taxable income, about $15,000 more than the 2026 brackets produce ($103,866 on $471,731 of ordinary income). It also used a $30,000 standard deduction instead of $32,200 and computed NIIT on $8,748 without subtracting the $3,000 allowable capital loss (the correct base is $5,748)."
-us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premiums from wages. The reference reduces wages only by the 401(k) deferral, so employment income is $499,073.84. It also used a $30,800 standard deduction instead of $32,200 and computed NIIT on $8,748 instead of the $5,748 base net of the capital loss."
-us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the expired pre-TCJA system: a $16,900 standard deduction, personal exemptions and higher pre-TCJA rates. The OBBBA made the TCJA brackets and the larger standard deduction permanent, so 2026 uses $32,200 with no exemptions. It also used $8,748 instead of $5,748 as the NIIT base."
-us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA had expired, using a $16,700 standard deduction plus $10,600 of personal exemptions and pre-TCJA rates. Under the OBBBA, 2026 uses the $32,200 standard deduction, no exemptions and the permanent 10–37% brackets. It also used $8,748 instead of $5,748 as the NIIT base."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rates and made up $45,620 of itemized deductions for mortgage interest and SALT that the facts do not include (no interest paid is listed, and Texas has no income tax), then applied a Pease limitation. The OBBBA 2026 law uses the $32,200 standard deduction and the permanent TCJA brackets."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its stated AGI of $476,323 conflicts with the $3,000 capital-loss limit, which gives AGI of about $504,779. Its $139,198 is about $35,000 above what the 2026 MFJ brackets produce on $472,578.56 of taxable income (the reference is $104,211.41 including qualified-dividend tax and NIIT)."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. The $123,753 is about $19,500 above the frozen reference's $104,211.41 and the exclusion's corrected value, $104,225.26, a gap that fits the pre-TCJA sunset rate schedule (33%/35% brackets and a small standard deduction) rather than the permanent 2026 OBBBA brackets and the $32,200 standard deduction."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a TCJA sunset that did not happen, using a $15,300 standard deduction, $10,100 of personal exemptions and pre-TCJA rates. It also subtracted the $16,208 ESI premiums from wages. The correct computation uses $499,073.84 of wages, the $32,200 OBBBA standard deduction and the 2026 10–37% brackets."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the head's $7,200 of interest and dropped the spouse's $700. It gave no bracket calculation, and its $105,744 is $1,533 above the frozen reference's $104,211.41 ($1,519 above the exclusion's corrected value, $104,225.26), a gap that reflects bracket thresholds below the 2026 schedule (the 32% rate starts at $403,550)."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working, and its $113,421 is about $9,200 too high. The frozen reference's path is $472,578.56 of taxable income taxed on the 2026 MFJ brackets for $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26. The overstatement comes from using a rate schedule heavier than the 2026 OBBBA brackets."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums, giving AGI of $488,614, then taxed that income at pre-TCJA sunset rates to reach about $117.5k of regular tax. Under the 2026 OBBBA brackets and the $32,200 standard deduction, regular tax on the frozen reference's $472,578.56 of taxable income is $103,993; that taxable income takes the $43 IRA deduction, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly applied the TCJA sunset (personal exemptions, a smaller standard deduction and pre-TCJA brackets) and subtracted ESI premiums from wages. The OBBBA made the TCJA brackets and the higher standard deduction ($32,200 MFJ in 2026) permanent."
-us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used the expired pre-TCJA structure: a $15,900 standard deduction, phased-out personal exemptions and the 10/15/25/28/33/35% brackets. The OBBBA kept the TCJA brackets and set the 2026 MFJ standard deduction at $32,200. It also left out the $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It put AGI at about $505,800, about $1,000 too high, used a $32,400 standard deduction and estimated ordinary tax at about $105,200 instead of $103,866 on the 2026 brackets. It then left out the $218.42 NIIT and rounded to $105,000."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no bracket calculation, and its $50,504 is about half the correct liability. On $472,578.56 of taxable income, the 2026 MFJ brackets run into the 32% rate and give $103,865.78 of ordinary tax, plus $127.20 on qualified dividends and $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It said retirement contributions and health premiums wiped out the liability, but the traditional 401(k) deferral and IRA contribution total only about $969 against $500,000 of wages. Taxable income is still $472,578.56 in the frozen reference, whose tax is $104,211.41; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It reported only regular tax ($103,879.68 + $127.20) and left out the $218.42 net investment income tax (3.8% of $5,748 because MAGI is above $250,000), which is part of income tax before refundable credits. It also skipped the $43 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $218.42 NIIT and taxed the $848 of qualified dividends at 20% ($169.60) instead of 15%, even though taxable income is below the 2026 MFJ 20% threshold of $613,700. It also skipped the $43 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It computed NIIT as 3.8% of $8,748 in interest and dividends ($332) without subtracting the $3,000 deductible net capital loss. The correct base is $5,748, for $218.42. It also skipped the $43 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It showed no bracket arithmetic, and its $57,891 is about $46,000 too low. The 2026 MFJ brackets on $471,730.56 of ordinary income give $103,865.78, and it also left out the $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premiums from wages, giving AGI of $488,614. The reference reduces wages only by the 401(k) deferral ($499,073.84), so AGI is $504,778.56 and taxable income is $472,578.56. That $16,208 at 32% explains almost the whole $5,173 shortfall."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: the standard deduction, 2026 brackets, 15% on qualified dividends and $218.42 NIIT. The one difference is that it did not deduct the $43 traditional IRA contribution, which the exclusion's corrected value also disallows under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That amount taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It stopped at regular tax ($103,879.68 + $127.20 on qualified dividends) and left out the $218.42 NIIT owed on $5,748 of net investment income because MAGI is above $250,000. It also skipped the $43 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 employer-sponsored insurance premiums as a pretax reduction of wages, giving AGI of $488,614. The reference reduces wages only by the 401(k) deferral, so AGI is $504,778.56. The lost $16,208 at 32% explains the $5,173 shortfall."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $140,000 with no bracket calculation, about $35,800 too high. The 2026 MFJ brackets on $472,578.56 of taxable income give $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It projected post-TCJA sunset parameters ($16,600 standard deduction, personal exemptions, pre-TCJA rates) where the OBBBA 2026 law applies. It also subtracted the $16,208 ESI premiums from wages and computed NIIT on $8,748 instead of $5,748."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 standard deduction instead of the 2026 OBBBA MFJ amount of $32,200, which overstated taxable income by about $1,450 (about $464 of tax at 32%). It also skipped the $43 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA individual provisions had expired and used a $16,950 standard deduction, personal exemptions and pre-TCJA rates. The OBBBA made the TCJA brackets permanent and set the 2026 MFJ standard deduction at $32,200, so the tax is overstated by about $22,000."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages (the reference reduces wages only by the 401(k) deferral). It also used a $30,750 standard deduction instead of $32,200, used projected bracket thresholds instead of the 2026 schedule, and computed NIIT on $8,748 instead of $5,748."
-us,scenario_052,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It put ordinary tax on $471,774 at $104,035, about $155 more than the 2026 schedule gives (10% to $24,800, 12% to $100,800, 22% to $211,400, 24% to $403,550, then 32%, for $103,880). It also skipped the $43.28 IRA deduction that the reference subtracts."
-us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or reasoning for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"From the 60-hour schedule it claimed a $4,517.80 OBBBA qualified-overtime deduction, but the reference allows no overtime deduction for this $500,000 highly compensated earner: taxable income is AGI minus only the $32,200 standard deduction. It also left out the $218.42 NIIT. It skipped the $43 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference takes it."
-us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Its reasoning cites contradictory standard deductions ($16,100 and $30,800) and never shows a bracket calculation. It also leaves out the $218.42 NIIT, so its $104,050 is a rough estimate rather than the $103,865.78 + $127.20 + $218.42 derivation."
-us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI and ordinary tax match the reference exactly ($103,865.78), but it taxed the $848 of qualified dividends at 0%. The 0% rate ends at $98,900 of 2026 MFJ taxable income, so 15% ($127.20) applies. It also left out the $218.42 NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $29,922 capital loss instead of the $3,000 limit, applied the 32% rate to a made-up $182,326 slice, and taxed made-up LTCG and qualified dividends at 0%. It then submitted $165,473.58, which does not follow from its own $85,184.10 result."
-us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 35% rate to $218,948 using a made-up bracket structure, although 2026 MFJ income up to $512,450 is taxed at no more than 32%. It also left the $926 401(k) deferral in income, used a $30,000 standard deduction instead of $32,200, and treated the qualified-dividend rate differential as a credit."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $117,285, which does not match its own figure of $107,277. That figure was also too high: it used a $32,600 standard deduction instead of $32,200, mis-applied the 2026 joint brackets (ordinary tax of $106,932 instead of about $103,866), and did not deduct the $43.28 IRA contribution."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution under the §219(g) active-participant phase-out, so its AGI was $504,821.84 instead of the reference's $504,778.56. At the 32% rate, that $43.28 accounts for the whole $13.89 gap. Its standard deduction ($32,200), brackets, 15% dividend tax and $218.42 NIIT all match the reference."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $504,779 was right, but it used a $28,700 standard deduction instead of the 2026 OBBBA $32,200 and estimated tax with a rough flat ~23% rate instead of the 2026 joint brackets. Its final $103,241 does not follow from any of its own figures, and it never added the $127.20 qualified-dividend tax or the $218.42 NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $105,000 without working through the brackets. The correct calculation taxes $471,730.56 of ordinary income at the 2026 joint brackets ($103,865.78), then adds $127.20 on qualified dividends and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax, which is a payroll tax that the question puts under payroll_tax, to income tax. It also used outdated bracket guesses ($24,150, $98,200, $209,400, $400,000) and a $32,300 standard deduction, then raised its $107,040 subtotal to $110,997 without any computation."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It mentioned the $926 401(k) deferral but did not subtract it, giving an AGI of $505,748 instead of $504,778.56. It also used a $32,300 standard deduction, understated the 2026 joint bracket thresholds ($401,950 instead of $403,550 for the top of the 24% bracket, among others), and left out the $218.42 NIIT owed because MAGI exceeds $250,000."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket estimates applied to about $471,350 of ordinary income give roughly $103,800, but it reported $141,500, an arithmetic error of about $37,000. It also left out the $218.42 NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax to income tax, although that tax belongs under payroll tax. It also used the outdated pre-OBBBA $30,000 standard deduction instead of $32,200 and inflated the bracket thresholds with its own estimates."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the annual $926 401(k) and $43 IRA contributions as weekly amounts and multiplied them by 52, which created a $31,000 401(k) deduction and a $2,236 IRA deduction. It then raised its $94,754 result to $121,500 without any calculation."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution under the §219(g) active-participant phase-out, so its AGI was $504,822 instead of $504,778.56. At the 32% rate this accounts for the roughly $14 gap. Its standard deduction, brackets, 15% dividend tax and NIIT all match the reference."
+us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 standard deduction instead of $32,200 and reported $119,769 of ordinary tax, about $15,000 more than the 2026 joint brackets produce on roughly $474k. It also calculated NIIT on the gross $8,748 instead of $5,748, which is net of the $3,000 allowable capital loss."
+us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, but only the 401(k) deferral reduces employment income, which is $499,073.84. It also used a $30,800 standard deduction instead of $32,200, skipped the $43.28 IRA deduction, and calculated NIIT on $8,748 instead of the net $5,748."
+us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the TCJA-sunset rules: a $16,900 standard deduction, personal exemptions reduced by the phase-out, and pre-2018 brackets. OBBBA made the TCJA rates, the $32,200 joint standard deduction and the zero exemption permanent for 2026. It also calculated NIIT on $8,748 instead of $5,748."
+us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $16,700 standard deduction plus $10,600 of personal exemptions with pre-2018 rates. OBBBA keeps the $32,200 standard deduction, no exemptions, and a 32% top rate in this income range. It also calculated NIIT on $8,748 instead of $5,748."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It made up $45,620 of itemized deductions, including mortgage interest and state and local taxes, even though only a mortgage balance was given and Texas has no income tax. It also applied the Pease limitation and pre-TCJA rates and taxed qualified dividends at 20% instead of using the $32,200 standard deduction, OBBBA brackets and the 15% rate."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It said capital losses were capped at $3,000 but reduced AGI by about $28,926 of losses, reaching roughly $476,323 instead of $504,778.56. Its $139,198 result is also far above what the 2026 joint brackets produce on any taxable income near $470k."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no figures. Its $123,753 is about $19,500 above the correct amount, in line with the other models that used the expired pre-TCJA rate schedule and smaller standard deduction and exemptions instead of OBBBA's permanent 2026 brackets and $32,200 standard deduction."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-TCJA-expiration rules: a $15,300 standard deduction, $10,100 of personal exemptions and pre-2018 brackets, giving $122,573.78 of ordinary tax. It also subtracted the $16,208 ESI premiums from wages, lowering AGI to $488,614."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It claimed taxable income was above the top marginal bracket, but $472,578.56 falls in the 32% bracket, which runs to $512,450 for joint filers in 2026. It also counted only the head's $7,200 of interest. Its unsupported $105,744 is $1,533 too high."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It reported $113,421 without showing the brackets or deduction it used, about $9,200 above the correct amount. The correct result applies the 2026 OBBBA joint brackets to $472,578.56 of taxable income ($103,865.78) and adds $127.20 of dividend tax and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, giving an AGI of $488,614, and its regular tax of about $117,547 reflects pre-TCJA rates instead of the 2026 OBBBA brackets."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly used TCJA-sunset rules (personal exemptions, the smaller pre-2018 standard deduction and pre-2018 brackets) and also treated the $16,208 ESI premiums as reducing wages. OBBBA made the TCJA structure permanent, so the correct base is the $32,200 standard deduction applied to an AGI of $504,778.56."
+us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $15,900 standard deduction, partially phased-out exemptions, and the 10/15/25/28/33/35% brackets, which gave $131,672.30 of ordinary tax. It also left out the $218.42 NIIT. The 2026 OBBBA joint schedule with the $32,200 deduction gives $103,865.78 of ordinary tax."
+us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It overstated AGI at about $505,800 and used a $32,400 standard deduction. It treated qualified dividends as a $144 reduction instead of $127.20 of tax at 15%, left out the $218.42 NIIT, and rounded to $105,000."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $50,504 is less than half the correct liability. On $471,730.56 of ordinary taxable income, the 2026 joint brackets alone produce $103,865.78, so it badly mis-applied the rate schedule."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It claimed retirement contributions and health premiums wipe out the liability, but those amounts total under $25,000 against $500,000 of wages. Taxable income remains $472,578.56, and the tax is $104,211.41."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 3.8% Net Investment Income Tax ($218.42 on $5,748 of net investment income, since MAGI exceeds the $250,000 joint threshold), which is part of income tax before refundable credits. It also skipped the $43.28 IRA deduction, so its $104,007 reflects only regular tax and the qualified-dividend tax."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $848 of qualified dividends at 20% ($169.60) instead of 15% ($127.20), even though taxable income is far below the 2026 joint 20% threshold of $613,700. It also left out the $218.42 NIIT and skipped the $43.28 IRA deduction."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It calculated NIIT on the gross $8,748 of interest and dividends ($332.42) instead of $5,748, which is net of the $3,000 allowable capital loss ($218.42). It also skipped the $43.28 IRA deduction, so it ended $127.89 too high."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $57,891 is about $46,000 too low. The 2026 joint brackets give $103,865.78 of ordinary tax on $471,730.56, so it badly mis-applied the rate schedule, and it also left out the $218.42 NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 ESI premiums as a pre-tax reduction of wages, reaching an AGI of $488,614. Only the $926.16 401(k) deferral comes out of employment income ($499,073.84), and the $43.28 IRA contribution is deducted, so AGI is $504,778.56."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution, keeping AGI at $504,821.84 instead of $504,778.56. At the 32% rate that accounts for the whole $13.89 gap. Its brackets, 15% dividend tax and $218.42 NIIT match the reference."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 3.8% NIIT ($218.42 on $5,748 of net investment income, with MAGI far above $250,000) and skipped the $43.28 IRA deduction. Its $104,006.88 is regular tax plus dividend tax on $472,622 instead of $472,578.56 of taxable income."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 ESI premiums as pre-tax wage reductions, reaching an AGI of $488,614. Only the $926.16 401(k) deferral reduces employment income and the $43.28 IRA contribution is deducted, so AGI is $504,778.56 and taxable income is $472,578.56."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rough guess of $140,000 without applying any brackets, about $35,800 too high. The 2026 joint brackets on $471,730.56 give $103,865.78, plus $127.20 of dividend tax and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used projected post-TCJA rules (a $16,600 standard deduction, personal exemptions and pre-2018 rates) instead of OBBBA's permanent 2026 schedule. It also subtracted the $16,208 ESI premiums from wages and calculated NIIT on $8,748 instead of the net $5,748."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 joint standard deduction instead of the 2026 OBBBA $32,200 and skipped the $43.28 IRA deduction. This overstated taxable income by about $1,494, which at 32% explains most of its $545 overstatement."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $16,950 standard deduction, $5,184 of remaining personal exemptions and pre-2018 rates, which gave $125,993 of regular tax. OBBBA keeps the $32,200 deduction, no exemptions and the TCJA brackets for 2026."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, reaching an AGI of $488,571, and used a $30,750 standard deduction instead of $32,200. It also calculated NIIT on the gross $8,748 instead of $5,748, which is net of the $3,000 allowable capital loss."
+us,scenario_052,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It miscalculated ordinary tax as about $104,035. Under the 2026 joint brackets, its own $471,774 of ordinary income gives $82,048 + 32% × $68,224 = $103,879.68, a $155 arithmetic error. It also skipped the $43.28 IRA deduction."
+us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It imputed an $88,400 FLSA overtime premium from the 60-hour schedule and claimed a $4,517.80 qualified-overtime deduction, but a $500,000 earner is FLSA-exempt as highly compensated, so no overtime deduction applies. It also left out the $218.42 NIIT and skipped the $43.28 IRA deduction."
+us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It switched between a $16,100 and a $30,800 standard deduction instead of using the 2026 OBBBA $32,200, and reported taxable income of about $488,648 instead of $472,578.56. It never calculated the $218.42 NIIT or showed the 15% dividend layer."
+us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income ($472,579) and ordinary tax ($103,866) match the reference, but it taxed the $848 of qualified dividends at 0%. Taxable income is far above the 2026 joint 0% ceiling of $98,900, so the 15% rate applies ($127.20). It also left out the $218.42 NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $29,922 net capital loss from ordinary income, ignoring the §1211(b) $3,000 limit, and taxed qualified dividends at 0%. It then reported $165,473.58, which does not match its own $85,184.10 calculation."
+us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 35% rate starting at about $256,800, roughly the single-filer threshold. For 2026 joint filers the 32% bracket runs to $512,450, so 35% does not apply here. It also used a $30,000 standard deduction, did not subtract the 401(k) deferral, subtracted the capital loss a second time, and left out NIIT."
us,scenario_052,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. Medicare's standard age threshold is 65, and the prompt supplies no disability, end-stage renal disease, or other alternative eligibility basis, so the head is not eligible."
-us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"It correctly computed $11,439 + $7,250 + $2,250 = $20,939 on full wages and also worked out an ESI-excluded alternative of $20,558.11. It then submitted $12,820.40, which matches neither calculation or any combination of its components. The submitted value is an output inconsistency, not a tax rule."
-us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also applied the $200,000 single Additional Medicare threshold instead of the $250,000 MFJ threshold, giving $20,403.20. It then submitted an unsupported $38,335.50 that contradicts even that figure."
-us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That understates employee Social Security tax by $520.80 ($10,918.20 vs $11,439). Its Medicare and MFJ Additional Medicare components were correct."
-us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied the Additional Medicare Tax at the $200,000 single-filer threshold instead of the $250,000 threshold for married filing jointly. That charges 0.9% on $300,000 ($2,700) instead of $250,000 ($2,250), overstating the total by $450."
-us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"Its components ($11,439 SS, $7,250 Medicare, $2,250 Additional Medicare) sum exactly to $20,939. It then 'rounded' to an unsupported $21,118, so the submitted value contradicts its own correct arithmetic."
-us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After cycling through several guesses, it settled on the 2025 wage base of $176,100 for 2026 instead of the actual 2026 base of $184,500. That understates Social Security tax by $520.80, producing $20,418.20."
-us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It switched from the correct $184,500 wage base to $176,100 and applied the $200,000 single threshold instead of the $250,000 MFJ threshold, computing $20,868.20. It then submitted $16,856.10, which matches none of its own component sums."
-us,scenario_052,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a rounded $180,000 Social Security wage base instead of the 2026 base of $184,500. That understates employee Social Security tax by $279 ($11,160 vs $11,439)."
-us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,208 ESI premiums from gross wages as if they were paid through a pre-tax cafeteria plan, although no such plan is stated. The Medicare and Additional Medicare base should be the full $500,000. It also used a $181,000 wage base instead of $184,500."
-us,scenario_052,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $181,800 Social Security wage base instead of the 2026 base of $184,500. That understates Social Security tax by $167.40 ($11,271.60 vs $11,439), while its Medicare and MFJ Additional Medicare amounts were correct."
-us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a projected $171,300 Social Security wage base instead of the 2026 base of $184,500. That gives $10,620.60 instead of $11,439 for employee Social Security tax."
-us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It computed $18,958.50 of Additional Medicare Tax, when 0.9% × ($500,000 − $250,000) is only $2,250, a rate error of roughly eightfold. It also capped Social Security at an outdated base ($9,932.50 instead of $11,439)."
-us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Its $20,279 matches a $180,000 wage base ($11,160) plus Medicare and Additional Medicare computed on $483,792, meaning it treated the ESI premiums as pre-tax. The correct figures are the $184,500 base and the full $500,000 of Medicare wages."
-us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages to $483,792 by treating the $16,208 ESI premiums as pre-tax, understating Medicare plus Additional Medicare by $380.89. It also used a $172,200 wage base instead of the 2026 base of $184,500."
-us,scenario_052,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It applied the $200,000 single Additional Medicare threshold instead of the $250,000 MFJ threshold. Its $34,980 total also far exceeds the correct $11,439 + $7,250 + $2,250 = $20,939, so it overstated Social Security well beyond the $184,500 cap."
-us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Its $20,353 is $586 below the correct $20,939. That gap matches treating the ESI premiums as pre-tax (−$380.89 in Medicare and Additional Medicare) combined with a Social Security wage base of about $181,200 instead of $184,500."
-us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It computed Medicare and Additional Medicare on $483,792 by treating the $16,208 ESI premiums as pre-tax instead of using the full $500,000. It also used a $181,800 wage base ($11,271.60) instead of $184,500 ($11,439)."
-us,scenario_052,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"Its $20,387 closely matches $20,390.71, the total from Medicare taxes on ESI-reduced wages of $483,792 plus Social Security on a base of about $181,800. It should have used the full $500,000 of Medicare wages and the $184,500 base."
-us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the $926 traditional 401(k) deferral from Medicare wages, but elective 401(k) deferrals remain FICA wages under IRC §3121(v)(1)(A). It also used a $183,000 wage base instead of $184,500."
-us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said no Additional Medicare Tax applied because wages were below the joint threshold, but $500,000 far exceeds the $250,000 MFJ threshold. Its $38,897 total also implies Social Security was taxed well beyond the $184,500 cap."
-us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It returned $0, ignoring that the Head's $500,000 of W-2 wages is subject to employee Social Security ($11,439), Medicare ($7,250), and Additional Medicare Tax ($2,250)."
-us,scenario_052,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $16,208 ESI premiums as pre-tax cafeteria-plan deductions and cut Medicare wages to $483,792, although the facts state no such plan. The correct base is the full $500,000 ($7,250 Medicare and $2,250 Additional Medicare), not $7,014.98 and $2,104.13."
-us,scenario_052,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It excluded the $16,208 ESI premiums from Medicare wages ($483,792) when the full $500,000 of gross wages is the payroll tax base. That understates Medicare plus Additional Medicare by $380.89."
-us,scenario_052,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a stale wage base of about $168,000 and then reported a total of about $13,500, which drops most of the $7,250 Medicare and $2,250 Additional Medicare it listed. The correct sum is $11,439 + $7,250 + $2,250 = $20,939."
-us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"It correctly used the $184,500 wage base but treated the ESI premiums as pre-tax, computing Medicare ($7,015) and Additional Medicare ($2,104) on $483,792. Those taxes apply to the full $500,000 of wages."
-us,scenario_052,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"It reduced FICA wages to $499,074 by subtracting the traditional 401(k) deferral, but elective deferrals are still subject to Medicare tax and Additional Medicare Tax. That understates those two components by $22."
-us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $16,208 ESI premiums from wages to get $483,792 of FICA wages instead of using the full $500,000. It also used a projected $183,144 wage base instead of the 2026 base of $184,500."
-us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no answer to score."
-us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"It listed the correct components ($11,439 + $7,250 + $2,250) but added them wrongly to $30,939. It then submitted an unsupported $11,573 instead of the correct sum, $20,939."
-us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, reaching $20,418.20 repeatedly. It then submitted $32,846, which contradicts its own stated total."
-us,scenario_052,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of $184,500. It also added the $2,250 Additional Medicare Tax twice, putting it on top of a $20,418.20 subtotal that already included it."
+us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"The model's reasoning explicitly reached the correct $20,939: $11,439 Social Security, $7,250 Medicare and $2,250 Additional Medicare. It then submitted $12,820.40, which matches neither that total nor its own $20,558.11 alternative that excluded the insurance premiums. The submitted value is disconnected from any calculation the model showed."
+us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also applied the $200,000 single Additional Medicare Tax threshold instead of $250,000 for a joint return. Finally, it discarded its own $20,403.20 subtotal and submitted an unexplained $38,335.50."
+us,scenario_052,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model named the correct components but gave no numbers. Its $15,300 total is lower than Social Security ($11,439) plus regular Medicare ($7,250) alone. The figure looks like the 15.3% combined FICA rate times $100,000, not a sum of the actual $11,439 + $7,250 + $2,250 components."
+us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That understated Social Security tax by $520.80 ($10,918.20 vs $11,439). Its Medicare and Additional Medicare amounts were correct."
+us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model applied the $200,000 single-filer Additional Medicare Tax threshold to a married couple filing jointly, whose threshold is $250,000. That produced $2,700 instead of $2,250 and overstated the total by $450."
+us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly computed $11,439 + $7,250 + $2,250 = $20,939. It then arbitrarily 'rounded' to $21,118 and submitted that figure, which no calculation supports."
+us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After switching between projections, the model settled on the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That understated Social Security tax by $520.80; its Medicare and Additional Medicare amounts were correct."
+us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model dropped the correct 2026 wage base of $184,500 in favor of $176,100 and applied the $200,000 single Additional Medicare threshold instead of $250,000 for joint filers. It then submitted $16,856.10, which does not match its own $20,868.20 sum."
+us,scenario_052,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used an approximate Social Security wage base of $180,000 instead of the 2026 base of $184,500. That understated Social Security tax by $279 ($11,160 vs $11,439)."
+us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $16,208 employer-sponsored insurance premium from Medicare wages, but the facts give no pre-tax cafeteria-plan treatment, so all $500,000 is Medicare wages. It also used a $181,000 Social Security wage base instead of $184,500. Both choices understated the tax."
+us,scenario_052,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $181,800 Social Security wage base instead of the 2026 base of $184,500. That understated Social Security tax by $167.40 ($11,271.60 vs $11,439)."
+us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a projected $171,300 Social Security wage base instead of the 2026 base of $184,500. That understated Social Security tax by $818.40; its Medicare and Additional Medicare amounts were correct."
+us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's Social Security figure of $9,932.50 implies an outdated wage base of about $160,200 instead of $184,500. Its Additional Medicare Tax of $18,958.50 is far above the correct 0.9% × $250,000 = $2,250, so it misapplied the 0.9% rate."
+us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model named the right components, but its $20,279 total is $660 short of $20,939. With $7,250 Medicare and $2,250 Additional Medicare, the shortfall corresponds to a Social Security wage base of about $173,900 instead of the 2026 base of $184,500."
+us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model cut FICA wages to $483,792 by treating the $16,208 employer-sponsored insurance premium as pre-tax, but the facts never say that, so Medicare wages are the full $500,000. It also used a $172,200 Social Security wage base instead of $184,500."
+us,scenario_052,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model applied the $200,000 single Additional Medicare threshold instead of $250,000 for joint filers. Its $34,980 total also implies about $25,000 of Social Security tax, well above the 6.2% × $184,500 = $11,439 cap, so it did not limit Social Security tax to the wage base."
+us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $20,353 total is $586 below $20,939. That is consistent with an understated Social Security wage base of about $175,000 instead of $184,500 alongside the correct $7,250 Medicare and $2,250 Additional Medicare amounts."
+us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model treated the $16,208 employer-sponsored insurance premium as a pre-tax exclusion and computed Medicare and Additional Medicare tax on $483,792 instead of the full $500,000. It also used a $181,800 Social Security wage base instead of $184,500."
+us,scenario_052,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $20,387 total is $552 short of $20,939. That is consistent with a Social Security wage base of about $181,800 together with Medicare wages reduced below $500,000, instead of the $184,500 base and the full $500,000 Medicare wages."
+us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model subtracted the $926 traditional 401(k) deferral from Medicare wages, but elective 401(k) deferrals remain subject to FICA, so Medicare wages are the full $500,000. It also used a $183,000 Social Security wage base instead of $184,500."
+us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model claimed no Additional Medicare Tax applies because wages are below the joint threshold, but $500,000 exceeds $250,000 by $250,000 and generates $2,250. Its $38,897 total is also roughly double the correct employee-side figure and matches none of its stated components."
+us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $0 on the theory that payroll tax did not apply, but $500,000 of W-2 wages is fully subject to employee FICA. That means $11,439 Social Security, $7,250 Medicare and $2,250 Additional Medicare Tax."
+us,scenario_052,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model treated the $16,208 employer-sponsored insurance premium as pre-tax and computed Medicare and Additional Medicare tax on $483,792. The facts state no cafeteria-plan treatment, so Medicare wages are the full $500,000, giving $7,250 and $2,250."
+us,scenario_052,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model subtracted the $16,208 employer-sponsored insurance premium from Medicare wages ($483,792). Medicare and Additional Medicare tax apply to the full $500,000, which gives $7,250 and $2,250 instead of $7,014.98 and $2,104.13."
+us,scenario_052,payroll_tax,grok-4.3,llm_error,other,False,"The model listed Social Security on about $168,000 plus 1.45% of $500,000 plus 0.9% above the threshold but submitted about $13,500. Just the first two components it described already sum to roughly $17,700, so the total is an arithmetic failure. It also used an outdated wage base instead of $184,500."
+us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model excluded the $16,208 employer-sponsored insurance premium from FICA wages, though the facts never designate it as pre-tax. It therefore computed Medicare and Additional Medicare tax on $483,792 instead of $500,000, understating the total by $381."
+us,scenario_052,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"The model reduced FICA wages by the $926 traditional 401(k) deferral, but elective deferrals are still FICA wages. Medicare tax should be $7,250 on $500,000 and Additional Medicare Tax $2,250, not $7,237 and $2,241."
+us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $16,208 employer-sponsored insurance premium from FICA wages and used a made-up $183,144 Social Security wage base instead of the 2026 base of $184,500. Both choices understated Medicare, Additional Medicare and Social Security tax."
+us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for payroll_tax, so there was no answer to score."
+us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model computed all three components correctly ($11,439, $7,250 and $2,250) but added them up to $30,939 instead of $20,939. It then submitted an unrelated $11,573 after an invented 'net of spouse portion' adjustment."
+us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning totals $20,418.20, using the outdated $176,100 wage base instead of $184,500. It then submitted $32,846, which none of its stated components support."
+us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model counted the $2,250 Additional Medicare Tax twice: it added it again to a $20,418.20 subtotal that already included it. It also used the outdated $176,100 Social Security wage base instead of the 2026 base of $184,500."
us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, and it also deducted $950 of auto loan interest. That interest does not qualify, because the OBBBA deduction covers only loans on new, U.S.-assembled vehicles. Its own arithmetic then gave $5,335.60, but it submitted $6,224, which follows from none of its steps. The correct result is $5,903.11 on $50,868.68 of taxable income."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It deducted $950 of auto loan interest as if it were above the line. The OBBBA qualified vehicle loan interest deduction applies only to loans on new, U.S.-assembled vehicles, which these facts don't establish (the vehicle is worth $7,410). That cut taxable income to $49,919 instead of $50,868.68. As a result, the $468.68 that belongs in the 22% bracket above $50,400 was never taxed there, giving $5,742.28 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted the ESI premium from wages and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It then applied an invalid bracket formula that produced $12,761.60. The $5,744 it submitted matches none of those steps. The correct result is $16,100 off $66,968.68, taxed at 10%/12%/22% with breakpoints at $12,400 and $50,400, which gives $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It guessed a 2026 standard deduction of $16,600 instead of $16,100 and computed $5,796. It then raised that to $6,196 with no computation to support the change. The correct taxable income of $50,868.68 puts $468.68 in the 22% bracket, giving $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning got the right answer: a $16,100 standard deduction, taxable income of $50,869, the 10%/12%/22% brackets at $12,400 and $50,400, and a tax of $5,903. It then discarded that result and submitted $6,027, calling it rounding to bracket estimates, which has no basis in its own computation."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived $5,903 from $50,869 of taxable income using the 2026 brackets of 10% to $12,400, 12% to $50,400 and 22% above. It then submitted $6,379, a figure unconnected to that derivation."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed a standard deduction of $15,700 instead of the OBBBA 2026 amount of $16,100. It also used the 2025 brackets (10% to $11,925 and 12% to $48,475) instead of the 2026 breakpoints of $12,400 and $50,400. That put $2,794 in the 22% bracket instead of $468.68, giving $6,193 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached $50,869 of taxable income. It then built the brackets by inflating 2025 thresholds by 2% (about $12,164 and $49,445) instead of using the 2026 breakpoints of $12,400 and $50,400, and it gave an unexplained total of $6,154. The actual 2026 brackets put only $468.68 at 22%, for $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and brackets of 10% to $12,200 and 12% to $49,950. The correct 2026 figures are a $16,100 deduction and breakpoints of $12,400 and $50,400. Those errors raised taxable income to $51,319 and pushed $1,369 into the 22% bracket, giving $6,051.18 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, which cut AGI to $63,580. It also used a $15,400 standard deduction and a $12,225 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. Together these understated the tax at $5,537.10."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA sunsets in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% rates. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption, and rates of 10%, 12% and 22% at this income, giving $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules: an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the $16,100 deduction and the 10%/12%/22% rates permanent. It also subtracted the $3,389 ESI premium from the given wages."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It assumed a $15,300 standard deduction instead of $16,100 and gave no bracket computation. Its $3,939 is about $2,000 below the tax that its own taxable income would produce at 10%/12%/22%. The correct result is $1,240 + $4,560 + $103.11 = $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expires in 2026 and applied the pre-TCJA standard deduction, personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent. It also subtracted the ESI premium from wages to get an AGI of $63,580 instead of $66,968.68."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA sunset parameters: an $8,300 standard deduction, a $5,300 exemption and brackets of 10%, 15% and 25%. OBBBA kept the TCJA structure permanently, with a $16,100 deduction and 10%/12%/22% brackets. It also improperly subtracted the $3,389 ESI premium from wages."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no deduction or bracket figures, and $3,737 is about $2,200 below the correct amount. The correct result applies 10% to $12,400, 12% to $50,400 and 22% on $468.68 of the $50,868.68 left after the $16,100 standard deduction, for $5,903.11. Its number matches taxing only about $34,000 of income, which means it understated taxable income or used lower rates."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, which lowered taxable income to $47,480. That kept all income out of the 22% bracket, and a small arithmetic slip gave $5,459.10. With AGI equal to the $66,968.68 in wages, taxable income is $50,868.68 and the tax is $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $63,580 by subtracting the $3,389 ESI premium from the given wages, which cut taxable income to $47,480 and the tax to $5,449.60. The listed wages are the taxable wage base. The correct taxable income is $50,868.68, with $468.68 taxed at 22%."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used 2025 parameters for 2026: a $15,000 standard deduction and 12% bracket ending at $48,475. The 2026 figures are a $16,100 deduction and 12% bracket ending at $50,400. That put $3,494 in the 22% bracket instead of $468.68, giving $6,347.18."
-us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and brackets ending at $11,600 and $47,150. The 2026 values are $16,100, $12,400 and $50,400. That overstated taxable income at $52,369 and put $5,219 in the 22% bracket, giving $6,574.18."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It subtracted a basic CTC or other nonrefundable credit, but this single filer has no children or dependents, so no CTC or credit for other dependents applies. The tax after nonrefundable credits equals the full regular tax of $5,903.11, not $3,657."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It used approximate standard deduction and bracket levels and gave no figures. Its $5,118 is about $785 below the tax the 2026 schedule produces on $50,868.68 of taxable income. Applying the $16,100 deduction and breakpoints at $12,400 and $50,400 gives $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction instead of $16,100. It also misapplied the brackets: even its own $50,969 of taxable income produces $5,925.18 at 10% to $12,400, 12% to $50,400 and 22% above, not the $5,793 it gave. The correct taxable income of $50,868.68 gives $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It got the correct taxable income of $50,869 but reported $5,985, which means it used wrong bracket thresholds. The 2026 single schedule taxes $12,400 at 10% ($1,240), $38,000 at 12% ($4,560) and only $468.68 at 22% ($103.11), for $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $3,389 of ESI premiums from wages that were already given as the taxable wage figure, which lowered taxable income to $47,480 and the tax to $5,449.60. With AGI equal to the $66,968.68 in wages, taxable income is $50,868.68 and the tax is $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,000 instead of the 2026 amount of $16,100, which gave taxable income of about $52,000. Its $5,850 doesn't even match the 2026 brackets applied to that income, which yield about $6,250. The correct result on $50,868.68 is $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA sunset law: an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses a $16,100 standard deduction, no exemption and 10%/12%/22% rates, giving $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset: an $8,522 standard deduction, a $5,435 restored exemption and 10%/15%/25% brackets. OBBBA made the $16,100 standard deduction and the 10%/12%/22% rate structure permanent for 2026."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 standard deduction and brackets of 10% to $12,225 and 12% to $49,725. The 2026 values are a $16,100 deduction and breakpoints of $12,400 and $50,400. That put $1,094 in the 22% bracket instead of $468.68, giving $5,963."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law, with an $8,372 standard deduction and 15%/25% brackets. OBBBA made the TCJA structure permanent, with a $16,100 deduction and 10%/12%/22% brackets. That overstated taxable income at $58,597 and the tax at $9,073."
-us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It assumed TCJA expired and used an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the $16,100 deduction and 10%/12%/22% rates permanent. It also subtracted the $3,389 ESI premium from the given wages."
-us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2023 standard deduction of $13,850 instead of $16,100 and computed $6,397.98. It then submitted $4,272, which contradicts its own arithmetic. The correct result on $50,868.68 of taxable income is $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction and 2025 brackets (breakpoints at $11,925 and $48,475) instead of the 2026 values of $16,100, $12,400 and $50,400, and computed $6,127.18. It then submitted $8,252, which contradicts its own computation."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It understated taxable income at $49,779 and then subtracted a nonrefundable child or dependent credit. This single filer has no children or dependents, so no CTC or credit for other dependents applies. The tax is the full $5,903.11 on $50,868.68 of taxable income."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premium input from wages and also took a $950 OBBBA auto-loan-interest deduction, even though no qualifying new-vehicle loan facts are listed. Wages of $66,968.68 are AGI with no adjustments. The model then submitted $6,224, which does not match its own computed $5,335.60."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Took the $950 auto loan interest as an OBBBA qualified passenger vehicle loan interest deduction. That deduction requires a loan for a new, US-assembled vehicle originated after 2024, and none is listed here. Without it, taxable income is $50,868.68 and $468.68 falls in the 22% bracket, giving $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Subtracted the $3,389 ESI premiums from wages and used the 2024 standard deduction ($14,600) and 2023 bracket figures instead of 2026's $16,100 deduction and the $12,400/$50,400 thresholds. Its bracket arithmetic produced $12,761.60, and it then submitted an unrelated $5,744."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Treated the $3,389 ESI premium input as a pre-tax reduction to wages, cutting taxable income to $47,480 instead of $50,868.68. It then understated the bracket tax even on that base: $47,480 yields $5,449.60, not $4,700."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a $16,600 standard deduction instead of the 2026 single amount of $16,100, which kept taxable income just under the 22% bracket. It then arbitrarily raised its own $5,796 result to $6,196 without any computation."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Computed the correct $5,903 from $50,869 taxable income and the 2026 $12,400/$50,400 thresholds. It then discarded that result and submitted an unexplained $6,027 by appealing to 'rounding to bracket estimates'."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Correctly derived taxable income of about $50,869 and tax of about $5,903 under the 2026 brackets. It then replaced that with an unsupported $6,379 attributed to a vague '2026 brackets estimate'."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Used a guessed $15,700 standard deduction and 2025 bracket thresholds ($11,925/$48,475) instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. This pushed $2,794 into the 22% bracket instead of $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Got taxable income of $50,869 right but built brackets by inflating the 2025 thresholds 2% instead of using the actual 2026 thresholds ($12,400 and $50,400). It also reported $6,154, more than even those approximated brackets produce."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used a $15,650 standard deduction and bracket thresholds of $12,200/$49,950 instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. This overstated taxable income by $450 and taxed $1,369 at 22% instead of $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premiums from wages to get a $63,580 AGI and used a $15,400 standard deduction and a $12,225 10% bracket. The correct figures are wage AGI of $66,968.68, the $16,100 deduction, and a $12,400 10% bracket."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA sunset in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. The OBBBA made TCJA permanent, so 2026 has a $16,100 standard deduction, no exemption, and 10%/12%/22% brackets."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA expired and used an $8,300 deduction plus a $5,300 exemption with 15%/25% brackets instead of the permanent OBBBA amounts ($16,100 deduction and 10%/12%/22% brackets). It also wrongly subtracted the $3,389 ESI premiums from wages."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Used a $15,300 standard deduction instead of $16,100. Its $3,939 is far below what even its own $51,669 taxable income yields under 2026 brackets ($6,079), so it misapplied the bracket rates on top of the deduction error."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Assumed TCJA expiration and applied a pre-TCJA standard deduction, personal exemption, and 15%/25% brackets instead of 2026 OBBBA law ($16,100 deduction and 10%/12%/22% brackets). It also wrongly subtracted the $3,389 ESI premiums from wages."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied post-sunset parameters ($8,300 deduction, $5,300 exemption, 15%/25% brackets) even though the OBBBA made TCJA rates and the $16,100 2026 standard deduction permanent. It also reduced AGI by the $3,389 ESI premium input, which is not a wage reduction here."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no computation. The correct path is $66,968.68 wages minus the $16,100 deduction, which is $50,868.68 taxable, giving $5,903.11. Its $3,737 matches taxable income of only about $33,200 under 2026 brackets, so it understated taxable income or the bracket rates by roughly $17,000 of base."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premiums from wages, cutting taxable income to $47,480 instead of $50,868.68, and also slipped arithmetically ($47,480 gives $5,449.60, not $5,459.10). Wages are AGI as given, which puts $468.68 in the 22% bracket."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Treated the $3,389 ESI premium input as a pre-tax reduction, so it applied the correct 2026 deduction and brackets to $47,480 instead of $50,868.68 taxable income. It never reached the 22% bracket."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the pre-OBBBA 2025 standard deduction ($15,000) and 2025 bracket thresholds ($11,925/$48,475) for 2026 instead of $16,100 and $12,400/$50,400. This overstated taxable income and the amount taxed at 22%."
+us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied 2024 parameters (a $14,600 standard deduction and $11,600/$47,150 thresholds) instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. That taxed $5,219 at 22% instead of $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Subtracted a 'basic CTC/nonrefundable credit' for a single 25-year-old with no children or dependents, who qualifies for no nonrefundable credit. Tax of $5,903.11 on $50,868.68 taxable income should stand unreduced."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Gave no computation. The correct path is wages minus the $16,100 deduction, which is $50,868.68 taxable, giving $5,903.11 under the 10%/12%/22% brackets. Its $5,118 matches only about $44,700 of taxable income, so it understated the base or the bracket rates by roughly $6,000 of taxable income."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used a $16,000 standard deduction instead of $16,100 and misapplied the 2026 brackets. Its own $50,969 taxable income yields $5,925.18 under the $12,400/$50,400 thresholds, not $5,793."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Reached the correct $50,869 taxable income but misapplied the bracket thresholds. $5,985 implies the 22% bracket starting near $49,600 instead of $50,400, which overstates the 22% slice above the correct $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Subtracted $3,389 of ESI premiums from gross wages as pre-tax contributions, giving $47,480 taxable instead of $50,868.68. That removed the $468.68 taxed at 22% and lowered the 12% slice."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Used a roughly $15,000 standard deduction instead of 2026's $16,100, giving about $52,000 taxable instead of $50,868.68. The resulting $5,850 is a loose approximation that does not follow the 2026 bracket arithmetic, under which even $52,000 would yield about $6,250."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed a TCJA sunset with an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets. Under the OBBBA, 2026 law keeps a $16,100 standard deduction, no exemption, and 10%/12%/22% rates."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset law (an $8,522 deduction, a $5,435 personal exemption, and 15%/25% brackets) instead of the permanent OBBBA 2026 parameters ($16,100 deduction, no exemption, 10%/12%/22% brackets)."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Used a $16,150 deduction and thresholds of $12,225/$49,725 instead of the 2026 values of $16,100 and $12,400/$50,400. That taxed $1,094 at 22% instead of $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Applied pre-TCJA law with an $8,372 standard deduction and 15%/25% brackets, leaving $58,597 taxable. Under the OBBBA, 2026 uses a $16,100 deduction and 10%/12%/22% brackets, giving $50,868.68 taxable."
+us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Assumed TCJA expiration with an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets instead of the permanent OBBBA amounts ($16,100 deduction and 10%/12%/22% brackets). It also subtracted the $3,389 ESI premiums from wages."
+us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Used the 2023 $13,850 standard deduction instead of 2026's $16,100, giving $6,397.98 on $53,119 taxable. It then submitted an unrelated $4,272 that contradicts its own computation."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Used a $16,000 deduction and 2025 thresholds ($11,925/$48,475) instead of $16,100 and $12,400/$50,400, getting $6,127.18. It then submitted $8,252, which matches none of its stated arithmetic."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Reduced tax by a child tax credit/dependent credit for a single filer with no children or dependents, who qualifies for no nonrefundable credits. It also used a wrong taxable income of $49,779 instead of $50,868.68."
us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component."
us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied the correct 6.2% Social Security and 1.45% Medicare rates to a base of $63,580. That base came from subtracting the $3,389 employer-sponsored insurance premium, which treats the premium as a pre-tax Section 125 exclusion the household facts never state. FICA applies to the full $66,969 of gross wages, and 7.65% of that is $5,123.10."
us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model explicitly deducted the $3,389 employer-sponsored insurance premium as a pre-tax payroll exclusion ($66,969 - $3,389 = $63,580) before applying 7.65%. The payroll-tax wage base is the full $66,969 of gross wages, so the correct total is $4,152.06 Social Security plus $971.05 Medicare, or $5,123.10."
@@ -4466,91 +4545,93 @@ us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model set up
us,scenario_053,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $63,580 'after pretax employer-plan premiums'. That subtracts the $3,389 employer-sponsored insurance premium as a cafeteria-plan exclusion the household facts do not state. FICA is levied on the full $66,969 of gross wages, which gives $5,123.10."
us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model defined taxable payroll wages as gross wages minus the $3,389 employer-sponsored insurance premium ($63,580), then applied 6.2% and 1.45% and rounded to $4,864. The premium is not excluded from the payroll-tax base. On the full $66,969 of wages, Social Security is $4,152.06 and Medicare is $971.05, for $5,123.10 in total."
us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate instead of the 2026 flat rate of 5.3%. It also taxed a $53,000 base built from an invented $13,969 exemption instead of $66,969 − $16,100 = $50,869, and it never removed the $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to its entire taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also subtracted the $950 auto loan interest from AGI, so AGI came out as $66,019 instead of $66,969. The frozen reference's tax is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"The model used Idaho's repealed graduated brackets (1%–6.5%, plus an invented 7.15% tier) instead of the flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also used a $7,200 standard deduction instead of the $16,100 federal-conforming amount, subtracted the ESI premiums, and invented a $59 exemption credit."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"The model used the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100. It never applied the $4,920 zero-rate bracket, and it dropped the correct 5.3% rate for an arbitrary lower effective rate, landing on $2,548 instead of 0.053 × $45,949 = $2,435."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a stale $4,673 zero-bracket threshold instead of 5.3% above $4,811, the threshold the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. It then raised its own $2,631 intermediate result to $2,867 with no basis."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"The model named the 5.3% rate and a roughly $4,900 threshold but never subtracted the threshold. Its $2,650 equals 5.3% of about $50,000. It also claimed the grocery credit offsets tax as a nonrefundable credit, but that credit is refundable and plays no part in this output."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"The model used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. That taxed $109 more income at 5.3% than the frozen reference, $5.79 of tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate with no zero-rate bracket instead of 5.3% above the zero-rate threshold ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also subtracted the $3,389 ESI premiums from AGI and used a $15,000 standard deduction instead of $16,100 on the full $66,969 AGI."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a $15,000 standard deduction, which gave $2,960. It then raised that to $3,706 with no basis. The frozen reference's computation is 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by 0.053 × $4,920 = $260.78."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"The model invented a 5.59% rate and a $4,000 personal exemption, which Idaho does not allow, and used a $15,650 standard deduction. Idaho instead applies 5.3% to taxable income above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference) after the $16,100 standard deduction; the exclusion's corrected value also subtracts the $5,000 of premiums, assuming they are not paid pre-tax."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to its full $48,180 base with no $4,920 zero-rate bracket. That base implies it subtracted the $3,389 ESI premiums from AGI and used a $15,400 standard deduction instead of $16,100."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"The model used an $8,300 standard deduction instead of the $16,100 single standard deduction that Idaho conforms to. On top of that, it applied 5.3% to the full $58,669 without subtracting the $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to a $49,980 base with no zero-rate bracket. That base reflects subtracting the ESI premiums from AGI and adding a nonexistent personal exemption. The frozen reference's computation is 5.3% × ($50,869 − $4,920); the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"The model gave no derivation. Its $2,652 matches 5.3% applied to roughly $50,000 of taxable income with no $4,920 zero-rate bracket removed. The frozen reference's result is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"The model applied a superseded rate of about 5.69% to all of roughly $49,852 of taxable income instead of 5.3% on only the income above the zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference)."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"The model used Idaho's 2023 rate of 5.8% instead of the 2026 rate of 5.3%, applied it with no zero-rate bracket, and invented a personal exemption. It also reduced AGI to $63,580 by subtracting the ESI premiums, though AGI is the full $66,969 of wages."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"The model gave no derivation. Its $3,450 is about 6.8% of the frozen reference's $50,869 base, so it either used a rate above Idaho's flat 5.3% or skipped most of the $16,100 standard deduction. Either way, it applied no $4,920 zero-rate bracket; the frozen reference's tax is $2,435, and the exclusion's corrected value is $2,176.06."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to all $47,480 of its taxable income with no $4,920 zero-rate bracket. It also subtracted the $3,389 ESI premiums from AGI, though Idaho starts from the full $66,969 AGI."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to about $47,470 of taxable income, a base that reflects subtracting the ESI premiums from AGI. It also omitted the $4,920 zero-rate bracket; the frozen reference's computation is 5.3% × ($50,869 − $4,920), and the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a $15,000 standard deduction instead of 5.3% and $16,100. It also taxed all $51,969 without removing the $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"The model invented a '$1,250 per $1,000 of federal taxable income' deduction that wiped out taxable income. The frozen reference subtracts only the $16,100 standard deduction, leaving $50,869, which it taxes at 5.3% above the engine's projected $4,920 for $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"The model used a 5.6% rate instead of Idaho's 2026 flat rate of 5.3%. It also applied the rate to the full $50,869 without subtracting the $4,920 single zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"The model gave no derivation and cited nonexistent 'state brackets'. Its $2,050 is an effective rate of about 4.0% on the $50,869 base, well short of the frozen reference's 5.3% × $45,949 = $2,435 (above the engine's projected $4,920 zero bracket) and of the exclusion's corrected value, $2,176.06."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"The model gave no computation. Its $1,195 is an effective rate of about 2.3% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"The model used stale parameters: the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100, and a $4,489 zero-rate threshold instead of the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026 (the frozen reference uses the engine's projected $4,920). Both enlarged the income taxed at 5.3%, by $672 against the convention's threshold and $781 against the frozen reference."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"The model used a zero-rate threshold of $4,946, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $135 less income at 5.3% than the $4,811 threshold does, and $26 less than the frozen reference ($1.36 of tax)."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"The model subtracted the $5,000 of health insurance premiums from Idaho income, which the exclusion's corrected value subtracts under Idaho Code 63-3022P (assuming the premiums are not paid through a pre-tax salary reduction) and the frozen reference, AGI minus only the $16,100 standard deduction, leaves out. It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Together these taxed $4,891 less income than the frozen reference; its $2,176.07 is $0.013 above the exclusion's corrected value, $2,176.057."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"The model applied 'progressive' rates to the $50,869 taxable income, but Idaho has a single flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). Its effective rate of about 4.25% understated the tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"The model used a zero-rate threshold of about $4,800, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, taxing $46,069 at 5.3% instead of $46,058 under the convention or $45,949 in the frozen reference. That put its tax $6.38 above the frozen reference and $0.60 above the $2,441.06 that the $4,811 threshold gives before the premium subtraction."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"The model subtracted the $3,389 ESI premiums from AGI, which lowered taxable income to $47,480 instead of $50,869. It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"The model used Idaho's 2023 rate of 5.8% instead of the 2026 flat rate of 5.3%, applied it to about $52,000 with no $4,920 zero-rate bracket, and understated the $16,100 standard deduction."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"The model used an $8,300 standard deduction instead of the $16,100 single standard deduction that Idaho conforms to. It then applied the superseded 5.695% rate to all $58,669 with no $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"The model used an $8,522 standard deduction instead of $16,100. It then applied the superseded 5.695% rate to all $58,447 without removing the $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"The model used a $16,150 standard deduction instead of $16,100 and a $4,942 zero-rate threshold, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, understating the income taxed at 5.3% by $72 against the frozen reference and $181 against the convention. It then rounded down, landing $4.28 below the frozen reference's $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"The model used the obsolete 5.8% rate instead of 5.3% and applied it to all $58,597 of taxable income. That base implies a standard deduction of about $8,400 instead of $16,100, and the model omitted the $4,920 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the output is missing rather than substantively computed."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78."
-us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"The model's reasoning skipped the $16,100 standard deduction entirely, applied the superseded 5.695% rate to all $66,969 of AGI, and used no zero bracket. It then submitted $2,878, which does not match its own $3,814 computation."
-us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 and omitted the $4,920 zero-rate bracket. It also subtracted the $10 Permanent Building Fund as a nonrefundable credit, but that fund is a separate $10 tax, not a credit, and it is not part of this liability."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"The model used the obsolete 5.8% rate and subtracted the ESI premiums from wages. It then arrived at $2,702 through an unexplained adjustment instead of computing the frozen reference's 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"The model gave no computation. Its $1,588 is an effective rate of only about 3.1% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06."
-us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Stated that PolicyEngine treats Idaho's grocery credit as nonrefundable and that $66,969 wages exceed refundable-credit income limits. The grocery credit is modeled as a refundable state credit with no income limit, so the full $155 base belongs in state_refundable_credits."
-us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Idaho has no refundable income tax credits at this income level, which overlooks the Idaho grocery credit entirely. That credit is refundable, has no income test, and pays $155 to this under-65 full-year resident."
-us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Named the grocery credit but reasoned that it only offsets Idaho tax liability, so nothing counts as refundable. The grocery credit is classified as a refundable credit whatever the liability, so its full amount counts. The model also used the stale $120 figure instead of the 2026 amount of $155."
-us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Called the $120 grocery credit 'nonrefundable in effect' because it offsets income tax and zeroed it out. The credit counts in full as a refundable state credit, and its 2026 value is $155, not $120."
-us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the grocery credit as applied against liability, leaving no refundable credit. The Idaho grocery credit is a refundable credit counted in full in state_refundable_credits: $155 for this one-person household."
-us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,Asserted that Idaho's grocery credit is nonrefundable and that Idaho has no refundable credits. The grocery credit is refundable and yields $155 for this under-65 full-year resident.
-us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Ruled out any 'grocery credit refund beyond offset' and cited income level and no dependents. The grocery credit is a per-person refundable credit with no income or dependent requirement, and it counts in full at $155."
-us,scenario_053,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,Correctly identified the refundable Idaho grocery credit for one person but used the outdated $120 per-person amount. The 2026 base for a filer under 65 is $155.
-us,scenario_053,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Claimed the grocery credit is fully phased out at $66,969 AGI. The Idaho grocery credit has no income phase-out, so the full $155 base applies."
-us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Zeroed the grocery credit by applying an income limit that does not exist. Every full-year Idaho resident gets the credit regardless of income, and it is $155 for this filer."
-us,scenario_053,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,Recognized the refundable grocery credit for the head but used the pre-2026 $120 amount instead of the 2026 per-person base of $155.
-us,scenario_053,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Identified the under-65 grocery credit but used $150 per person. The 2026 base amount is $155.
-us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,Estimated the refundable grocery credit at $120 per person for 2026. That is the outdated rate; the 2026 base is $155.
-us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no refundable state credits apply and never considered Idaho's refundable grocery credit. Every full-year resident receives that credit, and it is $155 here."
-us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,Correctly treated the grocery credit as refundable but used the stale $120 under-65 amount instead of the 2026 amount of $155.
-us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied a $100 per-person grocery credit, which is the pre-2023 rate. The 2026 base is $155."
-us,scenario_053,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Said no state refundable credits apply and omitted Idaho's universal refundable grocery credit, which pays $155 to this under-65 full-year resident."
-us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,Used $120 per eligible resident for the refundable grocery credit. The 2026 per-person base is $155.
-us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,Used the old $100 under-65 grocery credit amount instead of the 2026 base of $155.
-us,scenario_053,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Applied a $100 per-resident grocery credit, a figure superseded twice. The 2026 base amount is $155."
-us,scenario_053,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,Asserted that Idaho's grocery credit is nonrefundable and returned 0. The grocery credit is a refundable state credit worth $155 for this one-person household.
-us,scenario_053,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Labeled $120 the 'post-2025 amount' of the refundable grocery credit. The 2026 per-person base is actually $155, so the model applied the stale 2023-2025 rate."
-us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded that the household facts indicate no refundable Idaho credits, which overlooks the grocery credit. That credit needs only full-year Idaho residency and pays $155 here."
-us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Looked only for child-related refundable credits and returned 0, which misses Idaho's per-person refundable grocery credit of $155."
-us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,Correctly identified the grocery credit as the only refundable Idaho credit but used $120 instead of the 2026 nonelderly amount of $155.
-us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Claimed that income is too high for the Idaho grocery credit. The credit has no income test, so this full-year resident receives the full $155."
-us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the grocery credit at the outdated $120 per person; the 2026 base is $155. Its reference to a phaseout threshold is also wrong, since the credit has no income phaseout."
-us,scenario_053,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Returned 0 without considering Idaho's refundable grocery credit, which pays $155 to this under-65 full-year resident."
-us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,Correctly treated the grocery credit as refundable but used the pre-2023 $100 non-elderly amount instead of the 2026 base of $155.
-us,scenario_053,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,Paid a $100 grocery credit to a non-elderly adult. The 2026 per-person amount is $155.
-us,scenario_053,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,Used $120 for the non-senior refundable grocery credit instead of the 2026 base of $155.
-us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded that no state refundable credits apply, which omits Idaho's universal refundable grocery credit of $155 for this resident."
-us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,Applied the grocery credit at 'about $120' for one resident. The 2026 per-person base is $155.
-us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_refundable_credits, so there is no answer to score. The correct derivation is the $155 Idaho grocery credit for one under-65 full-year resident."
-us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly checked full-year eligibility and the SNAP and incarceration reductions, but used the stale $120 base. The 2026 under-65 amount is $155."
-us,scenario_053,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Considered only child-based credits and the property tax credit and never applied Idaho's refundable grocery credit. That credit pays $155 to every full-year resident regardless of children or rent.
-us,scenario_053,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,Stated a 2026 grocery credit of $120 per qualified member. The 2026 per-person base is $155.
-us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Checked only for a state EITC or child tax credit and concluded that no refundable credits apply. That misses Idaho's refundable grocery credit, worth $155 for this household."
-us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Said Idaho provides no refundable credit for this household and overlooked the refundable grocery credit. Every full-year resident receives it, and it is $155 here."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate instead of Idaho's 2026 5.3% rate. It also taxed an invented $53,000 base built from a made-up '$13,969 exemption', instead of $50,868.68 minus the $4,920 zero bracket ($45,948.68)."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"Got the 5.3% rate and $16,100 standard deduction right, but left out the $4,920 single zero bracket. It also lowered AGI to $66,019, which matches subtracting the $950 auto loan interest even though that interest does not reduce Idaho AGI. So it taxed $49,919 instead of $45,948.68."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"Used Idaho's repealed six-bracket schedule (1%–7.15%) and a $7,200 standard deduction instead of the flat 5.3% rate above a $4,920 zero bracket with the $16,100 standard deduction. It also subtracted the $3,389 ESI premium from income and invented a $59 personal exemption credit."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"Named the 5.3% flat rate on post-standard-deduction income but never did the calculation; $2,400 is a rough round number. The correct result taxes only $45,948.68, the amount of the $50,868.68 taxable income above the $4,920 zero bracket, giving $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"Used a $15,750 standard deduction instead of the 2026 amount of $16,100 and never applied the $4,920 zero bracket. After computing about $2,715 at 5.3%, it switched to a guessed 4.7% rate to get $2,548."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate and the 2024 $4,673 zero bracket instead of 5.3% above $4,920. It then dropped its own computed $2,631 and reported $2,867 with no basis."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"Its stated method (5.3% on $50,869 above a ~$4,900 zero bracket) gives about $2,436, but it reported $2,650, a figure its own setup doesn't produce. It also cited a grocery-credit offset, but the grocery credit is refundable and does not enter tax before refundable credits."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"Used the 2025 single zero bracket of $4,811 instead of the inflation-indexed 2026 amount of $4,920. That taxed $109 too much at 5.3%, about $5.79 over."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate with no $4,920 zero bracket instead of 5.3% above the zero bracket. It also subtracted the $3,389 ESI premium from AGI and used a $15,000 standard deduction instead of $16,100."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate, a $15,000 standard deduction and no zero bracket. It then threw out its own $2,960 result and reported $3,706 with no derivation; the correct figure is 5.3% × ($50,868.68 − $4,920)."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"Applied 5.3% to the full $50,869 of taxable income and left out Idaho's $4,920 single zero bracket, which cuts the tax by $260.76."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"Used a made-up 5.59% rate, a $15,650 standard deduction and a nonexistent $4,000 Idaho personal exemption. Idaho actually applies the $16,100 standard deduction and taxes income above a $4,920 zero bracket at 5.3%."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"Its $48,180 taxable income matches subtracting the $3,389 ESI premium and a $15,400 standard deduction, when AGI is unreduced and the deduction is $16,100. It also applied 5.3% without the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"Used an $8,300 standard deduction instead of the conforming 2026 single amount of $16,100. It also applied 5.3% to all taxable income, ignoring the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket, overstating the tax by about $261."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate and no zero bracket. Its $49,980 taxable income comes from subtracting the ESI premium and a nonexistent personal exemption instead of just the $16,100 standard deduction from the unreduced $66,968.68 AGI."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"Gave no derivation. $2,652 is roughly 5.3% applied to about $50,000 with no zero bracket, whereas the correct base is $45,948.68: $50,868.68 minus the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"Used a made-up 5.692% rate on about $49,852 with no zero bracket, instead of 5.3% on $45,948.68: the $50,868.68 taxable income minus the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"Used the 2023 5.8% rate instead of 2026's 5.3% and left out the $4,920 zero bracket. It also cut AGI to $63,580 by subtracting the ESI premium, which does not reduce Idaho AGI here."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"Gave no derivation. $3,450 matches an outdated ~5.7–5.8% rate on about $60,000 of taxable income, which understates the $16,100 standard deduction and skips the $4,920 zero bracket in the correct 5.3% calculation."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate with no zero bracket. It also subtracted the $3,389 ESI premium to get $63,580 AGI, when Idaho taxable income is $66,968.68 minus $16,100."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Applied the outdated 5.695% rate to about $47,470, which matches subtracting the ESI premium as well as the standard deduction. It never applied the 5.3% rate above the $4,920 zero bracket to the $50,868.68 taxable income."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"Used a $15,000 standard deduction instead of $16,100 and the old 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"Invented a '$1,250 per $1,000 of income' Idaho deduction that wipes out all taxable income. Idaho actually subtracts only the $16,100 standard deduction, leaving $50,868.68 taxed at 5.3% above the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"Used a made-up 5.6% rate on the full $50,869 instead of 5.3% on the part above the $4,920 single zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"Gave no rate or base, and $2,050 is too low. The correct calculation taxes $45,948.68 ($66,968.68 − $16,100 − $4,920 zero bracket) at 5.3%, giving $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"Gave no derivation, and $1,195 is less than half the correct tax. That matches a much lower effective rate than Idaho's flat 5.3% on the $45,948.68 above the zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"Got the structure right (5.3% above a zero bracket) but used a stale $15,750 standard deduction instead of $16,100 and an outdated $4,489 zero bracket instead of the 2026 indexed $4,920."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out Idaho's $4,920 single zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"Estimated the 2026 single zero bracket at $4,946 instead of $4,920 (and rounded taxable income to $50,869). That taxed about $26 too little and came in $1.36 low."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket that Idaho taxes at 0%."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"Subtracted a $5,000 health-insurance amount that is not part of Idaho taxable income here, which is just AGI minus the $16,100 standard deduction. It also used the 2025 $4,811 zero bracket instead of 2026's $4,920."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"Applied a progressive bracket schedule to $50,869. Idaho now taxes everything above the $4,920 single zero bracket at a flat 5.3%, so the correct figure is $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"Used a zero bracket of about $4,800 (its base of $46,069 is $50,869 − $4,800) instead of the 2026 indexed $4,920, which taxed about $120 too much at 5.3%."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"Started from $47,480 taxable income by subtracting the $3,389 ESI premium from wages, but AGI is the unreduced $66,968.68. It also used the 2025 $4,811 zero bracket instead of $4,920."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"Used the old 5.8% rate on about $52,000 with no zero bracket, instead of 5.3% on the $45,948.68 of taxable income above $4,920."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"Used an $8,300 standard deduction instead of 2026's $16,100 and the old 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"Used an $8,522 standard deduction instead of $16,100 and the outdated 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"Got the structure right but overestimated both indexed parameters, using a $16,150 standard deduction instead of $16,100 and a $4,942 zero bracket instead of $4,920. That shrank the taxed base to $45,877 and left it about $4 low."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Applied the repealed 5.8% rate to $58,597, which uses a deduction of only about $8,400 instead of $16,100. It also left out the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out Idaho's $4,920 single zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"Skipped the $16,100 standard deduction and applied the old 5.695% rate to the full $66,969 AGI, getting $3,814. It then reported a different $2,878 with no derivation, and it never used the 5.3% rate above the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 with no $4,920 zero bracket. It then subtracted a made-up $10 'Permanent Building Fund credit'; that fund is a $10 tax, not a credit, and it plays no part in this calculation."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Used the repealed 5.8% rate, subtracted the $3,389 ESI premium from wages, and reached $2,702 by an unexplained adjustment. The correct calculation is 5.3% on the $45,948.68 above the $4,920 zero bracket."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"Gave no computation, and $1,588 is far too low. Taxing the $50,868.68 of taxable income above the $4,920 zero bracket at 5.3% gives $2,435.28."
+us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model said PolicyEngine treats the Idaho grocery credit as nonrefundable and that the $66,969 wages exceed an income limit. In fact the credit is refundable, has no income test, and pays $155 for a full-year resident under 65 in 2026."
+us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said Idaho has no refundable credits at this income and never considered the grocery credit. That credit is refundable, has no income threshold, and pays $155 per resident under 65 in 2026."
+us,scenario_053,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 to this full-year resident under 65 in 2026 regardless of income, and answered $0."
+us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated the grocery credit as an offset against tax liability and so counted none of it as refundable. PolicyEngine counts the whole credit as a refundable state credit, at $155 for 2026, not the stale $120 the model cited."
+us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model called the grocery credit nonrefundable in effect because it offsets tax, and answered $0. The credit is fully refundable and counts in state_refundable_credits at $155 for 2026, not the outdated $120 the model cited."
+us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed the grocery credit is used up against liability, leaving no refundable remainder. The entire $155 grocery credit is a refundable credit, whatever the filer's tax liability."
+us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model wrongly called Idaho's grocery credit nonrefundable. The credit is refundable and pays $155 to a full-year resident under 65 in 2026.
+us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model ruled out the grocery credit because of income and the lack of dependents. The credit has no income limit and no dependent requirement, and it pays $155 to each full-year resident under 65 in 2026."
+us,scenario_053,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used the pre-2026 amount of $120. The 2026 amount for a resident under 65 is $155.
+us,scenario_053,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model invented an income phase-out for the Idaho grocery credit. The credit has no income limit or phase-out and pays the full $155 at $66,969 AGI."
+us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,The model assumed the grocery credit has an income limit that excludes this household. The credit has no income test and pays $155 to this full-year resident under 65.
+us,scenario_053,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used the outdated $120 per-person amount. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,The model identified the grocery credit correctly but used $150 per resident under 65. The 2026 base amount is $155.
+us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but estimated it at the pre-2026 $120 per person. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which every full-year resident gets regardless of income, and answered $0 instead of $155."
+us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used $120 for a grocery credit claimant under 65, which is the pre-2026 amount. The 2026 amount is $155."
+us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used an old $100 per-person grocery credit. The 2026 refundable amount for a resident under 65 is $155.
+us,scenario_053,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 to this full-year resident regardless of income, and answered $0."
+us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used the superseded $120 amount. The 2026 per-person amount is $155.
+us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model used an old $100 grocery credit for a single filer under 65. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,The model used a stale $100 per resident for the grocery credit. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model wrongly called Idaho's grocery credit nonrefundable. The credit is refundable and pays $155 to a full-year resident under 65 in 2026.
+us,scenario_053,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but used $120 and labeled it the post-2025 amount. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated refundable credits as zero because no specific facts pointed to one. Idaho's grocery credit needs only full-year residency, and it pays $155 here."
+us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked only for child-related refundable credits and missed Idaho's per-person refundable grocery credit, which pays $155 to this childless resident."
+us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model identified the grocery credit as the only refundable Idaho credit but used the pre-2026 $120 amount instead of $155.
+us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,The model claimed the income is too high for the grocery credit. The credit has no income test and pays $155 at any income level.
+us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used $120 instead of the 2026 amount of $155. It also cited a phase-out threshold that does not exist.
+us,scenario_053,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 to this full-year resident under 65 regardless of income, and answered $0."
+us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used an obsolete $100 amount. The 2026 amount for a non-elderly filer is $155.
+us,scenario_053,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,The model used an old $100 grocery credit for a non-elderly adult. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used the pre-2026 $120 amount. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 to every qualifying full-year resident under 65 in 2026, and answered $0."
+us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model identified the grocery credit for one resident but used the outdated $120 amount. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_refundable_credits, so there was no answer to score against the $155 reference."
+us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly applied the full-year grocery credit with no SNAP or incarceration reduction, but used the pre-2026 $120 amount. The 2026 amount is $155."
+us,scenario_053,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model looked only for child and property-tax credits and missed Idaho's refundable grocery credit, which pays $155 to this full-year resident."
+us,scenario_053,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model said the 2026 grocery credit is $120 per person. The 2026 amount is $155.
+us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model checked only for a state EITC and CTC, which Idaho lacks. It missed Idaho's refundable grocery credit, which pays $155 per resident under 65 in 2026."
+us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model said no Idaho refundable credit applies. It missed the refundable grocery credit, which pays $155 to this full-year resident regardless of income."
us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only a CHIP upper-income threshold and concluded that income below it establishes eligibility. It omitted the Medicaid-exclusion step: the child’s low income qualifies them for Medicaid under the OLDER_CHILD category, which makes CHIP eligibility false."
us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated being below North Carolina’s CHIP income limit and lacking other coverage as sufficient. It failed to test Medicaid eligibility first; the 10-year-old is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP.
us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from low income and the absence of employer-sponsored insurance. It omitted that low income places Child 1 in Medicaid’s OLDER_CHILD pathway, and Medicaid eligibility disqualifies the child from CHIP."
@@ -4568,50 +4649,52 @@ us,scenario_054,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibil
us,scenario_054,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model evaluated CHIP income limits and affordability of employer-sponsored insurance while omitting the prior Medicaid-eligibility exclusion. Child 2 qualifies for Medicaid in the young-child category, which categorically prevents CHIP eligibility; ESI affordability is irrelevant to this result."
us,scenario_054,child2_early_head_start_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It counted the $17,100 ""sstb self employment income before lsr"" input as household income, but that variable only flags what share of self-employment earnings comes from a specified service trade or business for the QBI deduction and does not populate self_employment_income, so the household's countable income for the Early Head Start poverty test is $0. It then misapplied the threshold it invoked: 100% of the federal poverty line for a three-person household is roughly $26,700, so even $17,100 sits below the 100% FPL income limit it cited. It also imposed a $14,000 bank-asset test, and Head Start and Early Head Start have no asset or resource test at all."
us,scenario_054,child2_early_head_start_eligible,gemini-3.5-flash-lite,llm_error,age_disability,False,"It inverted the Early Head Start age band, writing that age 2 ""exceeds the Early Head Start age limit of under 3"" — 2 is under 3, so the child sits squarely inside the birth-to-three infant/toddler band, and it is Head Start's 3-to-5 preschool band that excludes this child. That single reversed age comparison, not any income or categorical test, drove the 0."
-us,scenario_054,child2_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model applied the income test but omitted the minimum-age requirement for preschool Head Start. At age 2, Child 2 belongs to the Early Head Start age group and is not eligible for the requested Head Start output."
-us,scenario_054,child2_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly placed age 2 within the preschool Head Start age range. Preschool Head Start begins at age 3, so Child 2 fails the age test."
-us,scenario_054,child2_head_start_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model treated low household income and age 2 as sufficient but skipped the preschool Head Start minimum age. Income eligibility cannot overcome Child 2's failure to be at least age 3.
+us,scenario_054,child2_head_start_eligible,claude-haiku-5.5,llm_error,other,False,Its explanation correctly says that age 2 is below the preschool Head Start range of 3 to 5 and ends with 'value = 0'. It still submitted a numeric value of 1. This broke the required match between explanation and value and recorded an eligible answer that its own reasoning rejected.
+us,scenario_054,child2_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"It decided eligibility from the income test alone and never applied the preschool Head Start minimum age of 3. A 2-year-old is in the Early Head Start age band, not preschool Head Start, so Child 2 is ineligible no matter what the household earns."
+us,scenario_054,child2_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"It wrongly said age 2 falls inside the preschool Head Start age range. That range starts at age 3, and children under 3 are served only by Early Head Start, so Child 2 fails the age requirement."
+us,scenario_054,child2_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"It found eligibility because household income is below the Head Start threshold and skipped the age requirement. Preschool Head Start requires the child to be at least 3, so a 2-year-old is ineligible even with zero income."
us,scenario_054,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model applied CHIP's creditable-coverage bar to Medicaid, stating that a child with active employer-sponsored insurance is not Medicaid-eligible in North Carolina. Medicaid has no other-coverage exclusion — ESI affects CHIP eligibility, not the MAGI children's pathway — so the only test for a 2-year-old is the young-child income standard, which the household's 0.58 FPL clears by a wide margin."
us,scenario_054,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted the household's income level was too high for NC Medicaid, which inverts the actual comparison: household MAGI is 0.58 x FPL against North Carolina's children-ages-1-5 Medicaid standard of over 200% FPL. It benchmarked the child against an adult-style threshold (NC's parent/expansion-level cutoff) instead of the far more generous YOUNG_CHILD category limit that governs a 2-year-old."
us,scenario_054,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an affirmative unlisted categorical flag (disability, SSI, foster status) and defaulted to 0 when none appeared. Age alone establishes the YOUNG_CHILD MAGI category for a 2-year-old, leaving only the income test, and the household's 0.58 x FPL MAGI satisfies NC's children's standard, so the correct output is 1."
us,scenario_054,child2_wic_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model placed the age-2 child in WIC's child category correctly, then asserted the household ""exceeds WIC eligibility limits"" without applying the actual screen: WIC tests gross income against 185% of the federal poverty guideline, roughly $49,000 for a three-person household in 2026, and this household's income is at most the head's $17,100 of self-employment income (and $0 in the engine's WIC income measure) — under a third of the limit. It also skipped adjunctive eligibility, which qualifies the child through the household's Medicaid/SNAP/TANF eligibility even before the income screen is reached, and it disregarded the head's listed WIC receipt, which itself establishes the household was certified under the 185% FPL test."
us,scenario_054,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model answered from the absence of an explicit WIC-receipt flag on Child 2 rather than running WIC's eligibility test, applying the prompt's ""treat unlisted facts as false"" rule to a computed output instead of to inputs, in direct conflict with the instruction to report eligibility ""not whether they are currently enrolled."" Running the test yields eligibility: age 2 falls inside WIC's child category (ages 1 through 4), and household income far below 185% FPL (about $49,000 for three people) clears the income screen, with the head's own WIC receipt confirming household-level certification."
us,scenario_054,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model set the EITC at about $3,995, far below the two-child 40% phase-in amount of 40% x $15,891.93 = $6,356.77. It then left out the refundable CTC it had just computed ($2,190), reporting only the EITC as the total. Even that ACTC used gross $17,100 instead of earned income net of half the SE tax, so the correct refundable CTC is $2,008.79."
-us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model used the 92.35% net-earnings figure ($15,792) as earned income instead of SE profit minus half the SE tax ($15,891.93). It also computed the ACTC as 15% of the whole amount without subtracting the $2,500 threshold. It then threw away its own $8,686 sum and submitted $6,269, which roughly equals the EITC alone and leaves out the $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model confused itself about where $15,793 falls on the EITC schedule and settled on an EITC of about $2,293 (its $4,287 total minus its $1,994 ACTC). Earned income of $15,891.93 is inside the 40% two-child phase-in, so the EITC is $6,356.77. It also misstated SE tax as $2,233 and used the 92.35% figure instead of profit minus half the SE tax."
-us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household in the EITC phase-out region and cut the EITC to about $4,700. At $15,891.93 of earned income the credit is still phasing in at 40%, so it is $6,356.77. It also based the ACTC on gross $17,100 (about $2,190) instead of earned income net of half the SE tax, which gives $2,008.79."
-us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly found earned income of about $15,893. It then treated the EITC as near the plateau and used $6,164, instead of the 40% phase-in amount of $6,356.77. Its ACTC of $1,914 also misapplies its own formula: 15% x ($15,891.93 - $2,500) = $2,008.79."
-us,scenario_054,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used the 92.35% net-SE-earnings base ($15,791.85) as earned income. For the EITC and ACTC, self-employment earned income is net profit minus the deductible half of SE tax, $15,891.93. That understated the EITC ($6,316.74 vs $6,356.77) and the ACTC ($1,993.78 vs $2,008.79)."
-us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model got earned income ($15,891.93) and the EITC ($6,356.77) exactly right. It then used the pre-TCJA $3,000 ACTC threshold instead of the permanent $2,500 threshold in effect for 2026. That makes the refundable CTC $1,933.79 instead of $2,008.79."
-us,scenario_054,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income without subtracting the deductible half of SE tax, which inflated the EITC to $6,840 instead of $6,356.77. For the ACTC it used the expired $1,000-per-child credit and the $3,000 threshold. Under 2026 law the ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79, within the $1,700-per-child cap."
-us,scenario_054,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model took the 92.35% SE-tax base ($15,791.85) as earned income instead of SE profit minus half the SE tax ($15,891.93). That understated the EITC by about $40 and the ACTC by about $15, giving $8,311 instead of $8,365.56."
-us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted half the SE tax from the already-reduced 92.35% figure, arriving at $14,584. The correct earned income is $17,100 - $1,208.07 = $15,891.93, so this reduction was applied twice. It also used the obsolete $3,000 ACTC threshold instead of $2,500, understating both the EITC ($5,833.51 vs $6,356.77) and the ACTC ($1,737.57 vs $2,008.79)."
-us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model's EITC of $6,370 is close to the correct $6,356.77. Its total implies a refundable CTC of only about $497, when the ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79. That undercounts the refundable CTC by about $1,500."
-us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model computed earned income ($15,891.92) and the EITC ($6,356.77) correctly. It then applied the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold, getting $1,933.79 instead of $2,008.79."
-us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's EITC is correct (40% x $15,892). It applied the expired pre-TCJA ACTC rules: a $3,000 threshold and a $1,000-per-child credit. Under the 2026 rules ($2,500 threshold, $1,700-per-child refundable cap) the ACTC is $2,008.79, not $1,933.80."
-us,scenario_054,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model reported zero refundable credits. It missed that $17,100 of self-employment profit is earned income that qualifies a head of household with two qualifying children for a $6,356.77 EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $8,291 matches $6,356.77 EITC + $1,933.79 ACTC. So it got the correct earned income ($15,891.93) but used the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold, which gives $2,008.79."
-us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $8,290.56 is exactly $6,356.77 EITC plus an ACTC of 15% x ($15,891.93 - $3,000). It used the obsolete $3,000 ACTC threshold instead of the 2026 $2,500 threshold, which gives $2,008.79."
-us,scenario_054,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $8,291 matches the correct EITC of $6,356.77 plus an ACTC computed with the pre-TCJA $3,000 threshold ($1,933.79). The 2026 $2,500 threshold gives $2,008.79."
-us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,other,False,"The model reported only the refundable CTC and left out the EITC entirely, even though the head has $15,891.93 of earned income and two qualifying children, worth $6,356.77. Its ACTC also used gross $17,100 instead of earned income net of half the SE tax, giving $2,190 instead of $2,008.79."
-us,scenario_054,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the entire OBBBA CTC ($2,300 per child) as refundable. The refundable portion is limited to 15% of earned income over $2,500 and capped at $1,700 per child, which gives $2,008.79 here. It also reduced earned income twice to $14,584, understating the EITC ($5,834 vs $6,356.77)."
-us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model refused to credit any refundable amount even though the prompt says to assume filing. Self-employment profit of $17,100 with two qualifying children produces a $6,356.77 EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said it needed explicit eligibility indicators and returned 0, ignoring the instruction to assume filing and take-up. Self-employment earnings of $15,891.93 (net of half SE tax) with two qualifying children generate a $6,356.77 EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 of self-employment profit as earned income for both credits. Self-employment earned income excludes the deductible half of SE tax ($1,208.07), which gives $15,891.93. That overstated the EITC ($6,840 vs $6,356.77) and the ACTC ($2,190 vs $2,008.79)."
-us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model based both credits on gross $17,100 instead of earned income after subtracting the deductible half of SE tax ($15,891.93). That inflated the EITC to $6,840 (correct: $6,356.77) and the refundable CTC to $2,190 (correct: $2,008.79)."
-us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $5,834 EITC and $1,813 ACTC equal 40% and 15% x (E - $2,500) with E = $14,584. That base subtracts half the SE tax from the 92.35% figure, a double reduction. The correct earned income is $17,100 - $1,208.07 = $15,891.93, giving $6,356.77 + $2,008.79."
-us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model called the household one qualifying child plus one young child, when both children (ages 10 and 2) are qualifying children for the EITC and CTC. Its $5,978 is less than the two-child 40% phase-in EITC alone. The correct answer is $6,356.77 EITC plus a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model knowingly applied the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold. It also used the 92.35% figure ($15,792) as earned income instead of SE profit minus half the SE tax ($15,891.93). Both errors understate the credits: EITC $6,317 vs $6,356.77, ACTC $1,919 vs $2,008.79."
-us,scenario_054,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model computed the ACTC as 15% of earned income over $3,000. The 2026 threshold is $2,500. It also used $15,792 (the 92.35% SE base) as earned income instead of $15,891.93 (profit minus half the SE tax), so both the EITC and the ACTC ($1,919 vs $2,008.79) came out too low."
-us,scenario_054,federal_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income without subtracting half the SE tax, inflating the EITC to $6,840 instead of $6,356.77. It also assumed the CTC reverted to $1,000 per child with a $3,000 threshold in 2026. Under OBBBA's permanent rules, the refundable CTC is 15% x ($15,891.93 - $2,500) = $2,008.79."
-us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model based the EITC on gross $17,100 instead of earned income net of half the SE tax ($15,891.93), overstating it at $6,840 vs $6,356.77. It also capped the ACTC at a reverted $1,000 per child with a $3,000 threshold. The 2026 rules ($2,500 threshold, $1,700-per-child cap) give $2,008.79."
-us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model took gross $17,100 as earned income for the EITC ($6,840) instead of $15,891.93 net of half the SE tax ($6,356.77). It also set the refundable CTC at a flat $1,000 per child, when the 2026 ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79."
-us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so there was no answer to score."
-us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model worked out $7,932, using an understated earned income of $13,093 and a full $3,400 ACTC that ignores the 15%-over-$2,500 earnings limit. It then abandoned that figure and submitted an unrelated $672. The correct total is $6,356.77 EITC plus $2,008.79 ACTC, both based on $15,891.93 of earned income."
-us,scenario_054,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income for both credits. The deductible half of SE tax ($1,208.07) must come off first, leaving $15,891.93. That overstated the EITC ($6,840 vs $6,356.77) and the refundable CTC ($2,190 vs $2,008.79)."
-us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly claimed that SSTB self-employment income does not count as earned income for the EITC and ACTC. SSTB status matters only for the QBI deduction. Net SE earnings of $15,891.93 (after half the SE tax) qualify the household for a $6,356.77 EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"The model limited the ACTC to $375 on the premise that the CTC first offsets regular tax. After the head-of-household standard deduction, regular tax is $0, so the full 15% x ($15,891.93 - $2,500) = $2,008.79 is refundable. It also understated earned income at $15,522, and its $5,757 EITC does not even equal 40% of that base; the correct EITC is $6,356.77."
+us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It computed a refundable CTC of about $2,190 but left it out of the total and reported only an unsupported EITC of $3,995. The correct EITC is 40% phase-in × $15,891.93 earned income (SE profit less half SE tax) = $6,356.77, plus ACTC of $2,008.79."
+us,scenario_054,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It made EITC eligibility depend on income above the standard deduction and set refundable credits to $0. The EITC and ACTC are based on earned income ($15,891.93 of SE earnings), not taxable income, so they yield $6,356.77 + $2,008.79."
+us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly derived AGI of $15,892 after the half-SE-tax deduction, but then ran the credits on $15,792 and reached a combined $8,686. It then arbitrarily 'adjusted' that to $6,269, which drops the refundable CTC. Using $15,891.93 earned income gives EITC $6,356.77 plus ACTC $2,008.79."
+us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It placed earned income of about $15,800 incorrectly on the EITC schedule and guessed an EITC of about $2,300–3,000. That income is in the 40% two-child phase-in, so the EITC is 40% × $15,891.93 = $6,356.77. It also understated SE tax ($2,233) and based the ACTC on $15,793 instead of $15,891.93."
+us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated about $17,100 as being in the EITC phase-out region and cut the credit to about $4,700. Earned income of $15,891.93 (after the half-SE-tax deduction) is in the 40% phase-in, which gives $6,356.77. It also computed the ACTC on gross $17,100 instead of $15,891.93."
+us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the correct AGI of $15,893 but guessed an EITC of $6,164 instead of applying the 40% phase-in rate (40% × $15,891.93 = $6,356.77). It also miscomputed 15% × ($15,893 − $2,500) as $1,914 instead of $2,008.79."
+us,scenario_054,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used net SE earnings of $17,100 × 0.9235 = $15,791.85 as earned income. Earned income for the EITC and ACTC is SE profit minus the half-SE-tax deduction, which is $15,891.93, so the EITC is $6,356.77 and the ACTC is $2,008.79."
+us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Its EITC of $6,356.77 was correct, but it applied the pre-TCJA $3,000 ACTC earned-income threshold. Under the $2,500 threshold, which OBBBA made permanent, the ACTC is 15% × ($15,891.93 − $2,500) = $2,008.79, not $1,933.79."
+us,scenario_054,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used gross SE profit of $17,100 as earned income without subtracting the $1,208.07 half-SE-tax deduction, which inflated the EITC to $6,840 instead of $6,356.77. For the ACTC it applied the expired $1,000-per-child CTC and the $3,000 threshold instead of the 2026 rules, which give 15% × ($15,891.93 − $2,500) = $2,008.79."
+us,scenario_054,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It took earned income as $17,100 × 0.9235 = $15,791.85, the Schedule SE net-earnings base. The correct earned income is SE profit less half of SE tax, $15,891.93, which gives EITC $6,356.77 and ACTC $2,008.79."
+us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reduced income twice, applying the 92.35% factor and also subtracting half of SE tax, to get $14,584. Earned income is $17,100 − $1,208.07 = $15,891.93. It also used the expired $3,000 ACTC threshold instead of $2,500."
+us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its EITC of about $6,370 was close, but it allowed only about $497 of refundable CTC. With zero tax liability the ACTC is the full 15% × ($15,891.93 − $2,500) = $2,008.79, which is under the $3,400 cap."
+us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed the EITC correctly at $6,356.77 but applied the expired $3,000 ACTC threshold. With the $2,500 threshold, which OBBBA made permanent, the ACTC is $2,008.79 instead of $1,933.79."
+us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA ACTC rules: a $3,000 threshold and a $1,000-per-child limit. The 2026 rules use a $2,500 threshold and a $1,700-per-child refundable cap, which gives 15% × ($15,891.93 − $2,500) = $2,008.79 alongside the correct $6,356.77 EITC."
+us,scenario_054,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 with no derivation and missed that $15,891.93 of SE earned income with two qualifying children produces an EITC of $6,356.77 (40% phase-in) and a refundable CTC of $2,008.79."
+us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $8,291 equals EITC $6,356.77 plus an ACTC of $1,933.79, which comes from the expired $3,000 threshold. The $2,500 threshold gives an ACTC of $2,008.79."
+us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $8,290.56 equals the correct EITC of $6,356.77 plus 15% × ($15,891.93 − $3,000). That uses the pre-TCJA $3,000 ACTC threshold instead of the permanent $2,500 threshold, which gives $2,008.79."
+us,scenario_054,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $8,291 matches EITC $6,356.77 plus an ACTC computed above a $3,000 threshold. The ACTC threshold is $2,500, which makes the refundable CTC $2,008.79."
+us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,other,False,"It omitted the EITC entirely, even though $15,891.93 of SE earned income with two children yields $6,356.77 in the phase-in. It also computed the ACTC on gross $17,100 instead of on income net of the half-SE-tax deduction."
+us,scenario_054,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the CTC as fully refundable at $2,300 per child. Under OBBBA the CTC is $2,200 per child, and the refundable portion is limited to 15% of earned income over $2,500 ($2,008.79), with a cap of $1,700 per child. It also double-reduced earned income to $14,584 instead of $15,891.93."
+us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It declined to compute any credit and returned $0. The prompt assumes tax filing and take-up, so the head's $15,891.93 of SE earned income with two qualifying children produces EITC $6,356.77 and ACTC $2,008.79."
+us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It claimed there were no eligibility triggers and set credits to $0. Self-employment earnings and two qualifying children are sufficient for both credits, and filing is assumed, which yields an EITC of $6,356.77 and a refundable CTC of $2,008.79."
+us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used gross SE profit of $17,100 as earned income. EITC and ACTC earned income is net of the $1,208.07 half-SE-tax deduction, which gives $15,891.93, so the EITC is $6,356.77 instead of $6,840 and the ACTC is $2,008.79 instead of $2,190."
+us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It computed both credits on gross $17,100 and ignored the subtraction of the half-SE-tax deduction. Earned income is $15,891.93, which gives EITC $6,356.77 and ACTC $2,008.79."
+us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its EITC of $5,834 and ACTC of $1,813 both rest on earned income of $14,584. That figure applies the 92.35% factor and also subtracts half of SE tax. The correct earned income is $17,100 − $1,208.07 = $15,891.93."
+us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,other,False,"It gave an unsupported total of $5,978 and suggested that only one child counts fully. Both children are qualifying children, and $15,891.93 of earned income yields a two-child EITC of $6,356.77 plus ACTC of $2,008.79."
+us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the pre-TCJA $3,000 ACTC threshold instead of $2,500. It also used the 92.35% net-earnings figure of $15,792 instead of SE profit less half SE tax ($15,891.93), which understates both credits."
+us,scenario_054,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed the ACTC above a $3,000 threshold instead of the permanent $2,500 threshold. It also used $15,792 (net SE earnings at 92.35%) as earned income instead of $15,891.93, which gives $6,317 + $1,919 instead of $6,356.77 + $2,008.79."
+us,scenario_054,federal_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used gross $17,100 as earned income without subtracting the half-SE-tax deduction, which inflated the EITC to $6,840. It also assumed the CTC reverted to $1,000 per child with a $3,000 threshold in 2026, but OBBBA keeps the $2,500 threshold and a $1,700 refundable cap, which gives $2,008.79."
+us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took earned income as gross $17,100 instead of $15,891.93 after the half-SE-tax deduction, which produced an EITC of $6,840 instead of $6,356.77. It also applied the reverted $1,000-per-child CTC and $3,000 threshold, which OBBBA superseded."
+us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It computed the EITC on gross $17,100 instead of $15,891.93 after the half-SE-tax deduction, giving $6,840 instead of $6,356.77. It also set the ACTC at a flat $1,000 per child instead of 15% × ($15,891.93 − $2,500) = $2,008.79."
+us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for federal_refundable_credits, so there is no value to score."
+us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"Its reasoning reached $7,932 but it submitted an unrelated $672. Even that reasoning used understated earned income of $13,093 and granted the full $3,400 ACTC, ignoring the 15%-of-earned-income-over-$2,500 limit, which gives $2,008.79."
+us,scenario_054,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It treated gross SE profit of $17,100 as earned income. EITC and ACTC earned income subtracts the $1,208.07 half-SE-tax deduction, giving $15,891.93, so the credits are $6,356.77 and $2,008.79, not $6,840 and $2,190."
+us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It wrongly claimed that SSTB self-employment income is not earned income for the EITC. SSTB status matters only for the §199A QBI deduction, and SE earnings of $15,891.93 qualify for both the EITC ($6,356.77) and the ACTC ($2,008.79)."
+us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It cut the ACTC to $375 on the theory that the CTC first offsets regular tax. Regular tax is $0, because AGI of $15,891.93 is below the head-of-household standard deduction, so the full $2,008.79 is refundable. It also understated earned income ($15,522) and the EITC ($5,757 instead of 40% × $15,891.93 = $6,356.77)."
us,scenario_054,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that household income exceeds the free-meal threshold, but $17,100 for a three-person household is 63% of the 2026 federal poverty guideline — far below the 130% FPG free-meal cutoff (roughly $35,000 for a household of three) and even below the 185% reduced-price line. It applied a nonexistent income ceiling and never checked the SNAP categorical-eligibility pathway, which independently confers free meals on the 10-year-old school-age child regardless of the income test."
us,scenario_054,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model made free-meal eligibility contingent on an explicit eligibility flag or a positively stated SNAP amount in the household facts, when the benchmark required it to compute SNAP itself ($524.80/month here) and to apply the school-meal income test to the given $17,100 of self-employment income. It never evaluated either qualifying pathway — the 63%-of-FPG income share against the 130% free-meal threshold, or SNAP/TANF categorical eligibility for the 10-year-old K-12 student — and defaulted to 0 from the absence of a stated indicator rather than from a rule."
us,scenario_054,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model classified the head as a ""non-disabled childless adult without dependents"" when the household explicitly contains the head's two children (ages 10 and 2), which places the head in the parent/caretaker-relative category the engine assigned. It compounded this by asserting North Carolina has not expanded Medicaid, when NC implemented expansion to 138% FPL on December 1, 2023 under S.L. 2023-7 and that coverage is in force for 2026."
@@ -4620,117 +4703,121 @@ us,scenario_054,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligib
us,scenario_054,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model asserted the head's income exceeds NC's adult Medicaid threshold, but $17,100 of self-employment income for a household of three is roughly $15,900 of MAGI after the one-half self-employment tax deduction — 0.58 x FPL, less than half of North Carolina's 138% FPL expansion limit and well under the parent/caretaker limit. Its answer is consistent with comparing gross earnings against a limit near or below 100% FPL for a single person rather than the three-person FPL."
us,scenario_054,head_medicaid_eligible,minimax-m3,llm_error,other,False,"The model correctly derived the answer in its own explanation — NC covers adults to 138% FPL, ~$26,000 for a family of three, and $17,100 falls below it — and wrote ""Let me set to 1,"" then submitted value = 0. The submitted number contradicts its own completed income comparison, so the failure is in transcribing the derived result into the output field, not in the eligibility analysis."
us,scenario_054,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model identified the head as a parent and computed income at roughly 65% FPL, then discarded that result by asserting North Carolina has not expanded Medicaid and that its parent pathway demands ""very low income"" — NC expanded to 138% FPL effective December 1, 2023 (S.L. 2023-7), and the parent/caretaker category the engine applied is satisfied at 0.58 x FPL. It abandoned a qualifying pathway it had already established on a false statement about NC's expansion status."
-us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model counted having a child under 5 in the household as meeting the head's own WIC categorical requirement. In fact, an adult qualifies only if she is personally pregnant, postpartum or breastfeeding, and the head is none of these. The income test (185% of FPL) and asset test the model applied are irrelevant because the categorical test already fails, and WIC has no asset test at all."
-us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model called the head a 'qualifying caretaker' because Child 2 is age 2. WIC has no caretaker category: a parent applies on a child's behalf but is not a participant unless she is pregnant, postpartum or breastfeeding, and the head has none of those statuses."
-us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly listed the WIC categories (pregnant, postpartum, breastfeeding, infant, child under 5) but then credited the household's under-5 child to the head. The head is a 40-year-old with no pregnancy, postpartum or breastfeeding status, so she fits no category. The model also treated her reported WIC receipt as proof of eligibility."
-us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model inferred a postpartum or breastfeeding status for the head from the 'receives wic' flag. The prompt sets every unlisted status input to false, so the head is neither postpartum nor breastfeeding and fails the WIC categorical test regardless of income below 185% of FPL."
-us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the head as a 'categorically eligible caretaker' of the 2-year-old and cited adjunctive eligibility. Adjunctive eligibility satisfies only the WIC income test, never the categorical test. An adult is categorically eligible only when pregnant, postpartum or breastfeeding, and the head is none of these."
-us,scenario_054,head_wic_eligible,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model based eligibility only on WIC receipt and household income below 185% of FPL, and it skipped the categorical requirement. A 40-year-old head who is not pregnant, postpartum or breastfeeding has no WIC category, so income is never reached."
-us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated the 'receives wic' flag as proof of eligibility. That flag records the household's WIC take-up, which runs through the 2-year-old child. The head herself fails the categorical test because she is not pregnant, postpartum or breastfeeding."
-us,scenario_054,head_wic_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model relied on the reported WIC receipt and the income limit and never applied the categorical test. An adult qualifies only when pregnant, postpartum or breastfeeding, and the head has none of those statuses, so she is ineligible whatever her income."
-us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model took the head's reported WIC receipt as confirmation of eligibility and skipped the individual categorical test. The head is 40 and not pregnant, postpartum or breastfeeding, so she is in no WIC participant category. Only the 2-year-old qualifies."
-us,scenario_054,head_wic_eligible,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model combined reported WIC receipt with income eligibility and never checked whether the head fits a WIC category. She is not pregnant, postpartum or breastfeeding, so the categorical test fails before income is considered."
-us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model noted that no pregnancy or postpartum status was stated, which should have ended the analysis because unlisted statuses are false and an adult qualifies for WIC only in those statuses. It then overrode that finding with the receipt flag and the young child, and neither of those confers categorical eligibility on the head."
-us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model took 'Head receives WIC' as meaning the head is WIC-eligible. Eligibility requires the head herself to be pregnant, postpartum or breastfeeding, and she is none of these; the receipt comes through the eligible 2-year-old."
-us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model asserted eligibility only because the head is listed as receiving WIC and applied no categorical test. A 40-year-old adult with no pregnancy, postpartum or breastfeeding status fits no WIC category."
-us,scenario_054,head_wic_eligible,gpt-6-luna,llm_error,categorical_eligibility,False,"The model treated reported WIC receipt as proof of eligibility. WIC eligibility requires a personal categorical status, and the non-pregnant, non-postpartum, non-breastfeeding head does not have one."
-us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model said current WIC receipt confirms eligibility and never applied the categorical requirement. An adult qualifies only when pregnant, postpartum or breastfeeding, and the head has none of these statuses."
-us,scenario_054,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model made up a pregnant, postpartum or breastfeeding status for the head, but the household facts contain no such input and the prompt sets unlisted statuses to false. Without that status the head has no WIC category and is ineligible."
+us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Treated having a child under 5 in the household as the categorical test for the head. WIC has no parent or caretaker category, and a 40-year-old who is not pregnant, postpartum, or breastfeeding does not qualify. It then checked income and assets, which can't fix a categorical failure. WIC also has no asset test."
+us,scenario_054,head_wic_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Concluded the head is eligible because the 'receives wic' flag is set. That flag records receipt and does not establish eligibility. The head is a non-pregnant 40-year-old whose youngest child is 2, so the head is past the postpartum and breastfeeding windows and fits no WIC category."
+us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"Invented a 'qualifying caretaker' WIC category from Child 2 being under 5. WIC serves only pregnant, postpartum, or breastfeeding women, infants, and children under 5. The parent of a 2-year-old is not categorically eligible, whatever their income."
+us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"Correctly listed the WIC categories but never checked whether the head fits one. Instead it used the household's child under 5 plus the receipt flag as proof. A non-pregnant 40-year-old whose youngest child is 2 is outside the 6-month postpartum and 12-month breastfeeding windows, so the head is not eligible."
+us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"Assumed the head is a postpartum or breastfeeding participant based only on the receipt flag. No pregnancy or postpartum status is given, and the youngest child is 2, well past both windows. Having income below 185% FPL does not make the head eligible without a qualifying category."
+us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"Treated the head as a 'categorically eligible caretaker' of a child under 5. WIC has no such category. It also confused the receipt flag with adjunctive eligibility, which only satisfies the income test (through SNAP, Medicaid, or TANF) and never the categorical test."
+us,scenario_054,head_wic_eligible,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Relied on the receipt flag and household income below 185% FPL, skipping the person-level categorical test. A 40-year-old head who is not pregnant, postpartum, or breastfeeding fits no WIC category, so the head is ineligible even with qualifying income."
+us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Equated the 'receives wic' input with eligibility. The head is a non-pregnant 40-year-old with no infant under 12 months, so the head fails WIC's categorical requirement. Only Child 2 (age 2) is eligible."
+us,scenario_054,head_wic_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Combined the receipt flag with the income test and never applied the categorical requirement. The head is not pregnant, not within 6 months postpartum, and not breastfeeding a child under 1, so the head is not WIC eligible."
+us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Treated the 'receives wic' flag as confirming eligibility. Eligibility is assessed separately from receipt, and a 40-year-old adult who is not pregnant, postpartum, or breastfeeding fits no WIC category."
+us,scenario_054,head_wic_eligible,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Relied on the reported WIC receipt and on income eligibility alone. WIC also requires the individual to be pregnant, postpartum, breastfeeding, an infant, or a child under 5. The 40-year-old head meets none of these."
+us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Acknowledged that no pregnancy or postpartum status is stated but still called the head eligible because of the receipt flag and a young child in the household. The prompt says unlisted statuses are false, so the head has no qualifying category and is ineligible."
+us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Read the explicit 'receives wic' fact as an eligibility determination. It is only a receipt input, and the head fails WIC's categorical test because no pregnancy, postpartum, or breastfeeding status applies to a 40-year-old whose youngest child is 2."
+us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Asserted eligibility only from the receipt flag and never applied WIC's categorical criteria. Under those criteria, a non-pregnant adult parent of a 2-year-old is excluded."
+us,scenario_054,head_wic_eligible,gpt-6-luna,llm_error,categorical_eligibility,False,"Treated reported WIC receipt as equivalent to eligibility. The question asks for eligibility, and the head fits no WIC category: not pregnant, not within 6 months postpartum, and not breastfeeding an infant under 1."
+us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"Said receiving WIC confirms eligibility, so it skipped the person-level categorical test. A 40-year-old adult with no pregnancy and a youngest child aged 2 is not WIC eligible."
+us,scenario_054,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"Assumed without basis that the head is pregnant, postpartum, or breastfeeding. No such status is listed, so it is false under the prompt's rules, and the youngest child is 2, past both the postpartum and breastfeeding windows. The head therefore has no WIC category."
us,scenario_054,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages and that self-employment income is excluded from employee payroll tax, but then replaced the resulting zero with an invented $1,084 residual benchmark. No such residual employee payroll tax applies in North Carolina, so the stated computation yields $0."
us,scenario_054,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required payroll_tax output entirely. The derivation requires $0 because the household has no wages subject to employee Social Security, Medicare, Additional Medicare, or mandatory state employee payroll taxes."
us,scenario_054,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the 185% reduced-price ceiling as sufficient and failed to evaluate the superior free-meals tier. At 63% of the federal poverty guideline and with SNAP/TANF categorical eligibility, the household receives free meals, so reduced-price eligibility is 0."
us,scenario_054,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model reversed the income comparison by asserting that roughly $15,000 was above a free-meals threshold it estimated at roughly $32,500. The household's traced ratio is 63% of the federal poverty guideline, below the 130% free threshold, and SNAP/TANF categorical eligibility independently places it in the FREE tier, leaving no reduced-price support."
-us,scenario_054,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model made multiple arithmetic errors after correctly identifying the 92.35% net-earnings adjustment: $15,791.85 × 15.3% equals $2,416.15305, not $2,418.16 or $2,414.29. It also incorrectly invoked a deduction for half of the Social Security portion in computing the tax liability; the one-half-SE-tax deduction applies to adjusted gross income, not SECA tax."
-us,scenario_054,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model identified the correct formula but rounded inaccurately. Applying it without premature approximation gives $17,100 × 0.9235 × 0.153 = $2,416.15305, which rounds to $2,416.15 rather than $2,415.00."
-us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model replaced the exact SECA computation with an inaccurate rough estimate. The stated formula yields $17,100 × 0.9235 × 0.153 = $2,416.15305, rounded to $2,416.15, not $2,413.00."
-us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's own formula produces approximately $2,416, but it submitted $1,596 without a valid computation connecting the two amounts. Excluding Additional Medicare Tax does not reduce the ordinary 12.4% Social Security plus 2.9% Medicare SECA liability, which equals $2,416.15."
-us,scenario_054,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly described a result near $2,416.18 but submitted the inconsistent amount $2,419.95. Exact arithmetic using the unrounded base gives $15,791.85 × 15.3% = $2,416.15305, rounded to $2,416.15."
-us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the deduction for one-half of self-employment tax from the SECA liability and then produced a final number inconsistent with its intermediate figures. That deduction reduces adjusted gross income for income-tax purposes; it does not reduce self-employment tax, which remains $17,100 × 92.35% × 15.3% = $2,416.15."
-us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"Its net income (about $929) and $279 contribution were essentially right. It then projected a three-person maximum allotment of about $820 for FY2026 instead of the actual $785. That inflated the monthly benefit to about $540 ($6,480 a year) instead of $505."
-us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"It used the correct $785 maximum, $209 standard deduction and $931 net income. It then subtracted an unrounded $279.30 contribution. SNAP rounds 30% of net income up to the next whole dollar ($280), so the allotment is $505 a month ($6,060), not $505.70."
-us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It zeroed SNAP because the $14,000 bank balance exceeds a resource limit. It missed that North Carolina's broad-based categorical eligibility (TANF non-cash benefits) makes this household categorically eligible, so the balance does not disqualify it and it receives $505 a month."
-us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It cut countable income by half the self-employment tax (to about $1,324 a month), which SNAP does not do: the full $1,425 counts. It also used a $224 standard deduction and a $789 maximum instead of $209 and $785. It then inflated its own $6,456 result to $7,152 with an unexplained 'COLA adjustment'."
-us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"Its derivation reached about $508 a month ($6,096), close to the correct $505. It then dropped that result for an unsupported 'refined estimate' of $707 a month, which ignores the $280 expected contribution from $931 net income."
-us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It assumed an $800 three-person maximum instead of $785. Its $583 monthly estimate also implies a contribution of only about $217, not the $280 that is 30% of $931 net income rounded up. Together these overstate the annual benefit at $7,000."
-us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"It correctly derived $931 net income and the $785 maximum. It then subtracted an unrounded $279.30 contribution to get $505.70. SNAP rounds 30% of net income up to $280, which gives $505 a month and $6,060 a year."
-us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a 40% cost-of-doing-business cut to self-employment income, reducing gross income to $855 a month. No business expenses are listed, so the full $1,425 counts. It also used a $228 standard deduction and a $766 maximum. Its derivation gave $7,548, yet it submitted $5,532, which none of its stated arithmetic supports."
-us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"It stacked the 20% earned income deduction twice and used the FY2025 $768 maximum instead of $785. It then subtracted its roughly $280 contribution from $768 and got about $350 a month instead of $488. That arithmetic slip produced $4,200 instead of $6,060."
-us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"It used the correct $785 maximum and $931 net income. It then took 30% as an unrounded $279.30 and got about $505.70 a month. SNAP rounds the contribution up to $280, which gives $505 a month and $6,060 a year, not $6,068."
-us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted $2,416 of self-employment tax before counting income, which cut countable income to $1,223.65 a month. SNAP counts the full $1,425. It also used the outdated $198 standard deduction and a $775 maximum instead of $209 and $785, which inflated the benefit to $540.72 a month."
-us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $781 maximum and a $202 standard deduction instead of the FY2026 values of $785 and $209. That raised net income to $938 and cut the allotment to $499 a month ($5,988) instead of $505."
-us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,It used a $784 maximum and a $208 standard deduction instead of $785 and $209. It also left the 30% contribution unrounded at $279.60 rather than rounding it up to a whole dollar. The result was $504.40 a month instead of $505.
-us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its $931 net income was correct, but it used a $787 maximum instead of $785. It also subtracted an unrounded $279.30 instead of rounding the contribution up to $280, which gave $507.70 a month ($6,092) instead of $505."
-us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the FY2024 parameters: the $766 maximum, and net income of $942 from the $198 standard deduction. The FY2026 values are $785 and $209, which give $931 net income. The result was $483.40 a month instead of $505."
-us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no derivation. Its $3,720 implies $310 a month, which requires an expected contribution of about $475 against the $785 maximum. The correct contribution is $280: 30% of $931 net income after the $285 earned income deduction and the $209 standard deduction."
-us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $5,800.80 equals $483.40 a month, which is the FY2024 $766 maximum minus 30% of $942 net income (using the FY2024 $198 standard deduction). It did not use the FY2026 $785 maximum and $209 deduction, which give $505."
-us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,It used a $747 three-person maximum and the outdated $198 standard deduction instead of the FY2026 values of $785 and $209. That gave $464 a month instead of $505.
-us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It declared the household over the income limits, but $1,425 a month of gross income is 64% of the poverty guideline. $931 of net income is 42%. Both easily pass the gross and net income tests, so the household is owed $505 a month."
-us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $7,548 ($629 a month) matches a derivation that cuts self-employment income by a 40% business-cost deduction to $855 a month and uses a $766 maximum. SNAP counts the full $1,425 because no expenses are listed, and the maximum is $785. The correct result is $505 a month."
-us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,820 ($485 a month) matches outdated maximum allotment and standard deduction values of about $766–768 and $198–204. The FY2026 values are $785 and $209, which give a $280 contribution and $505 a month."
-us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $5,844 ($487 a month) exactly matches the FY2025 parameters: the $768 maximum, minus a contribution rounded up to $281 on $936 net income after the $204 standard deduction. It did not apply the FY2026 $785 maximum and $209 deduction, which give $505 a month."
-us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $228 monthly standard deduction ($2,736 a year) instead of $209, which understated the contribution at $273.60. It also used an $800 maximum ($9,600 a year) instead of $785. The benefit came out at $526 a month ($6,317) instead of $505."
-us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Its $931 net income and roughly $279 contribution were right, but it used an $816 maximum instead of the FY2026 figure of $785. That inflated the benefit to $537 a month instead of $505."
-us,scenario_054,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated countable income as 'very low' and awarded $739 a month, close to the full maximum. It never computed the $280 expected contribution that $931 of net income generates, which leaves the correct benefit at $505."
-us,scenario_054,snap,gpt-5.4-nano,llm_error,other,False,"It claimed the facts did not settle eligibility and defaulted to $0. It ignored the listed $17,100 of self-employment income and the three-person household, which are all the formula needs to produce net income of $931 and an allotment of $505 a month."
-us,scenario_054,snap,gpt-5.5,llm_error,other,False,"It set up the correct formula: the $785 maximum minus 30% of $931 net income. Then it miscalculated $785 − $279 as $575 instead of about $505. The correct figure, with the contribution rounded up to $280, is $505 a month ($6,060)."
-us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It applied a North Carolina resource limit to the $14,000 bank balance and zeroed SNAP. It missed that the household is categorically eligible through TANF non-cash benefits under broad-based categorical eligibility, so the balance does not disqualify it."
-us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"It correctly found $931 net income and eligibility under broad-based categorical eligibility. It then rounded the benefit to $506 instead of rounding the 30% contribution up to $280. $785 − $280 gives $505 a month ($6,060), not $506 ($6,072)."
-us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but rounded $505.70 to a $506 monthly benefit. It should have rounded the $279.30 contribution up to $280, which gives $505 a month and $6,060 a year."
-us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"Its formula, 12 × [785 − 0.30 × (1425 × 0.80 − 209)], left the 30% contribution unrounded at $279.30. SNAP rounds it up to the next whole dollar ($280), so the benefit is $505 a month and $6,060 a year, not $6,068.40."
-us,scenario_054,snap,gpt-6-luna,llm_error,other,False,"It computed $785 minus 30% of $931 as $505.70 without rounding the contribution up to $280. The correct allotment is $505 a month, or $6,060 a year."
-us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It claimed both income and assets exceed SNAP limits. Gross income is only 64% of the poverty guideline, and broad-based categorical eligibility in North Carolina means the $14,000 of assets does not disqualify the household. It is owed $505 a month."
-us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It used FY2024 parameters: a $766 monthly maximum ($9,192 a year) and a $198 standard deduction ($2,376 a year). The FY2026 values are $785 and $209, which give a $280 contribution and $505 a month instead of its roughly $483."
-us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"It used FY2025 parameters: a $768 maximum ($9,216 a year) and a $204 standard deduction ($2,448 a year). The FY2026 values are $785 and $209. It also left the contribution unrounded, and it got $487 a month instead of $505."
-us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 values of $785 and $209. It also took an unrounded 30% contribution, which produced $5,846.40 instead of $6,060."
-us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"Its contribution of about $280 was correct, but it assumed an $811 maximum instead of the FY2026 figure of $785. That inflated the benefit to $531 a month instead of $505."
-us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"It zeroed SNAP because the $14,000 bank balance exceeds a resource limit. It ignored North Carolina's broad-based categorical eligibility through TANF non-cash benefits, which means the balance does not disqualify the household, so the full $505 monthly allotment applies."
-us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the standard countable-resource limit for households without elderly or disabled members. The household is categorically eligible under North Carolina's broad-based categorical eligibility, so its $14,000 bank balance does not disqualify it."
-us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly derived $931 net income and rounded the contribution up to $280. It then used a $783 three-person maximum instead of the FY2026 figure of $785, which gave $503 a month ($6,036) instead of $505."
-us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It declared $17,100 of income over a 130% poverty limit of about $26,000, which is a mistaken comparison because $17,100 is below $26,000. Gross income is 64% of the poverty guideline, so the household passes the income tests and is owed $505 a month."
-us,scenario_054,snap,ox-alpha,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but subtracted an unrounded $279.30. SNAP rounds 30% of net income up to $280, so the benefit is $505 a month and $6,060 a year, not $505.70 a month."
-us,scenario_054,snap,qwen-3.7-max,llm_error,other,False,"It awarded a 'maximum' benefit of $934 a month, which skips the 30% net-income contribution entirely and overstates the maximum, which is $785. The $931 of net income generates a $280 contribution, which leaves $505 a month."
-us,scenario_054,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It misstated monthly earnings as $1,579 instead of $1,425. It also claimed net income exceeds the limit, but $931 of net income is only 42% of the $2,220.83 monthly poverty guideline, so the household passes the net income test and is owed $505 a month."
+us,scenario_054,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"The model dropped the correct 0.9235 × 15.3% product and moved to a garbled cap formula. In that formula it miscalculated both pieces: Medicare on $15,791.85 at 2.9% is $457.96, not $453.02, and Social Security at 12.4% is $1,958.19, not $1,961.27. Earnings are under the wage base, so SE tax is simply $15,791.85 × 0.153 = $2,416.15. Its own first-pass figure of $2,418.16 was also a multiplication error."
+us,scenario_054,self_employment_tax,claude-haiku-5.5,llm_error,other,False,"The model correctly set up $17,100 × 0.9235 × 15.3% and reached about $2,416. It then submitted $2,412 with no reason given, an unexplained downward change of about $4 from the correct $2,416.15."
+us,scenario_054,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The method was right: 15.3% of 92.35% of $17,100. But the model reported $2,415 instead of the exact product, $15,791.85 × 0.153 = $2,416.15, which is an arithmetic or truncation error."
+us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The model used the right formula, 92.35% of earnings × 15.3%, but multiplied wrong and got 'roughly $2,413'. The exact result is $15,791.85 × 0.153 = $2,416.15."
+us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed $17,100 × 0.9235 × 0.153 ≈ $2,414, which is close to correct. It then claimed that excluding Additional Medicare Tax and rounding gave $1,596. Additional Medicare Tax is $0 at these earnings, and nothing should be subtracted from regular SECA, so the answer is $2,416.15."
+us,scenario_054,self_employment_tax,minimax-m3,llm_error,other,False,"The model got net SE earnings of $15,792 and SE tax of about $2,416.18, which is essentially correct. It then submitted $2,419.95, a number that matches none of its own figures. The correct value is $15,791.85 × 0.153 = $2,416.15."
+us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model correctly reached $2,416 of SE tax. It then applied 0.9235 a second time and subtracted the deductible half of SE tax from the SE tax itself, producing $1,578. The half-SE-tax deduction is an above-the-line deduction against income tax and does not reduce SE tax, which stays at $2,416.15."
+us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"It projected a three-person maximum allotment of about $820 instead of the FY2026 value of $785, and used a $211 standard deduction instead of $209. As a result it got $540/month instead of $785 − $280 = $505."
+us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but kept 30% of net income as $279.30. SNAP rounds the expected contribution up to $280, so the allotment is $505/month ($6,060), not $505.70."
+us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a SNAP resource limit to the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives the asset test for this household, which is at 64% of the poverty line. The household is eligible for $505/month."
+us,scenario_054,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It zeroed out SNAP based on an asset and income test. Broad-based categorical eligibility removes the asset test, and $1,425/month of gross income passes easily. It also netted SE tax out of income, but SNAP counts the full $17,100 of self-employment earnings."
+us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted half of SE tax from self-employment earnings ($1,324/month instead of $1,425), which SNAP does not do. It also used a $224 standard deduction and a $789 maximum instead of $209 and $785. It then inflated its own $6,456 result to $7,152 with no basis."
+us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"Its first pass ($785 maximum minus about $277 = $508/month) came close to the correct $505. It then dropped that result for an unsupported $707/month 'refined estimate', which matches no SNAP calculation."
+us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It assumed an $800 maximum allotment and came up with $583/month without showing the work. With net income of $931, the contribution is $280 and the benefit is $785 − $280 = $505/month."
+us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were all correct ($1,425 gross, $285 earned income deduction, $209 standard deduction, $785 maximum), but it did not round 30% of net income ($279.30) up to $280. That gave $505.70/month instead of $505."
+us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a 40% self-employment cost deduction that has no basis here. No business expenses are listed, so SNAP counts the full $1,425/month. It also used a $228 standard deduction and a $766 maximum, and submitted $5,532 even though its own work concluded $7,548."
+us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"Even on its own figures (a $768 maximum minus about $280), the benefit comes to about $488/month, but it stated about $350/month. It also double-counted the 20% earned income deduction and used a $204 standard deduction instead of $209."
+us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"It computed $785 − 0.30 × $931 = $505.70 without rounding the expected contribution up to $280. The correct benefit is $505/month, or $6,060 a year."
+us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,416 of SE tax from self-employment earnings before computing SNAP income. SNAP counts the full $1,425/month. It also used an outdated $198 standard deduction and a $775 maximum instead of $209 and $785."
+us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,It used outdated parameters: a $781 maximum allotment and a $202 standard deduction instead of the FY2026 values of $785 and $209. That produced net income of $938 and $499/month instead of $931 net and $505/month.
+us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $784 maximum and a $208 standard deduction instead of $785 and $209. It also kept the contribution at $279.60 rather than rounding it up to a whole dollar, which gave $504.40/month instead of $505."
+us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the three-person maximum allotment as $787 instead of $785. It also left the contribution at $279.30 instead of rounding it up to $280, which gave $507.70/month instead of $505."
+us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,It used an old $766 maximum and a $198 standard deduction (net income $942) instead of the FY2026 values of $785 and $209 (net income $931). The result was $483.40/month instead of $505.
+us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $3,720 answer implies $310/month, which badly understates the benefit. The correct figure is the $785 maximum minus 30% of $931 net income, rounded up to $280, which equals $505/month."
+us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $5,800.80 matches the outdated $766 maximum and $198 standard deduction ($766 − 0.3 × $942 = $483.40). The FY2026 values are $785 and $209, which give $505/month."
+us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,It used a stale $747 three-person maximum allotment and a $198 standard deduction instead of $785 and $209. That gave $464/month instead of $505.
+us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It claimed income exceeds the SNAP limits, but $1,425/month of gross income is 64% of the $2,220.83 monthly poverty guideline. That is far below both the gross and net income tests, so the household gets $505/month."
+us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $7,548 ($629/month) matches cutting self-employment income by a 40% cost allowance to $855/month and using a $766 maximum. SNAP counts the full $1,425/month with no listed expenses, so the benefit is $505/month."
+us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $485/month matches an outdated $766 maximum minus a contribution of about $281. The FY2026 parameters give a $785 maximum minus a $280 contribution, which is $505/month."
+us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $487/month matches the FY2025 parameters: a $768 maximum and a $204 standard deduction, giving net income of $936 and a $281 contribution. The FY2026 values are $785 and $209, which give $505/month."
+us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $228/month standard deduction and an $800/month ($9,600/year) maximum allotment instead of $209 and $785. That produced $6,317 instead of 12 × ($785 − $280) = $6,060."
+us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,It computed net income of $931 correctly but overstated the three-person maximum allotment as $816 instead of $785. That gave $537/month instead of $505.
+us,scenario_054,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated countable income as 'very low' and awarded about $739/month. The household has $931 of net income after deductions, which creates a $280 expected contribution against the $785 maximum, for $505/month."
+us,scenario_054,snap,gpt-5.4-nano,llm_error,missing_output,False,"It defaulted SNAP to 0 without running the benefit calculation, even though it had household size, self-employment income, and state. The standard calculation gives $505/month."
+us,scenario_054,snap,gpt-5.5,llm_error,other,False,"It set up the right formula ($785 − 30% × $931) but got $575 by mistake. The correct arithmetic, with the $279.30 contribution rounded up to $280, gives $505/month."
+us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It applied a resource limit to the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives the SNAP asset test for this low-income household."
+us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"It got net income of $931 and correctly noted that broad-based categorical eligibility removes the asset test. It then rounded the $505.70 benefit to $506 instead of rounding the $279.30 contribution up to $280, which gives $505/month."
+us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,It rounded $785 − $279.30 to $506 instead of rounding the 30% contribution up to $280 first. The correct monthly allotment is $505.
+us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"Its formula 12 × [785 − 0.30 × (1425 × 0.80 − 209)] leaves the contribution at $279.30. SNAP rounds it up to $280, which gives $505/month and $6,060 a year."
+us,scenario_054,snap,gpt-6-luna,llm_error,other,False,It used $505.70/month because it did not round 30% of the $931 net income up to $280. The correct allotment is $505/month.
+us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It claimed both income and assets exceed the limits. Gross income is 64% of the poverty line, and broad-based categorical eligibility waives the asset test, so the household gets $505/month."
+us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It used an outdated $766/month maximum ($9,192/year) and a $198/month standard deduction ($2,376/year) instead of the FY2026 values of $785 and $209. That gave $5,801 instead of $6,060."
+us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"It used the FY2025 parameters: a $768 maximum and a $204 standard deduction ($2,448/year). The FY2026 values are $785 and $209, which give a $280 contribution and $505/month."
+us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"It used the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 values of $785 and $209. It also did not round the contribution up to whole dollars, which gave $5,846.40 instead of $6,060."
+us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It overstated the three-person maximum allotment as about $811 instead of $785, and used a standard deduction of about $205 instead of $209. That gave $531/month instead of $505."
+us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"It ruled the household ineligible because of a resource limit on the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility removes the asset test."
+us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied a countable resource limit for households without elderly or disabled members. That limit does not apply here, because North Carolina's broad-based categorical eligibility waives the asset test for a household at 64% of the poverty line."
+us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"It handled net income ($931), the rounded-up $280 contribution, and the asset waiver correctly, but used a $783 maximum allotment instead of the FY2026 three-person value of $785. That gave $503/month instead of $505."
+us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It got the comparison backwards. $17,100 of annual income is below its own roughly $26,000 limit at 130% of the poverty line, so the household passes the gross income test and gets $505/month."
+us,scenario_054,snap,ox-alpha,llm_error,other,False,It used all the correct FY2026 inputs but kept the expected contribution at $279.30 instead of rounding it up to $280. That gave $505.70/month instead of $505.
+us,scenario_054,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It awarded a $934 'maximum' benefit with no income reduction. The three-person maximum is $785, and the household's $931 of net income creates a $280 contribution, which leaves $505/month."
+us,scenario_054,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used $1,579 of monthly earnings instead of $1,425 and claimed net income exceeds the limit. In fact, $931 of net income is 42% of the poverty line, so the household gets $505/month."
us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly applied North Carolina's $19,125 head-of-household standard deduction against federal AGI of $15,892 and stated twice in its own reasoning that NC taxable income floors at $0 and ""NC tax = 0,"" then discarded that conclusion and submitted $245. No NC computation produces $245 from these facts: at the 2026 flat rate of 3.99% it implies $6,140 of taxable income, which the household does not have once the standard deduction is applied."
us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly computing $0 NC taxable income using the head-of-household standard deduction of $19,125, the model reassigned the filer to single status and its $12,750 standard deduction despite two qualifying children in the household, yielding $3,142 of taxable income and $133.50 of tax. It then submitted $596, a figure tied to neither derivation and roughly 3.75% of gross self-employment income with no standard deduction at all."
us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied the $12,750 single-filer standard deduction rather than the $19,125 head-of-household deduction the head qualifies for with two dependent children, and used the 2025 rate rather than the 2026 flat rate of 3.99%: $15,892 − $12,750 = $3,142 taxed at 4.5% is $141.39, matching its $142. Under the correct head-of-household deduction the base is zero and the tax is $0."
us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model applied a rate directly to gross self-employment income with no standard deduction and no half-SE-tax adjustment: $342 is exactly 2% of the $17,100 SSTB figure. North Carolina's 2026 flat rate is 3.99% and it applies to taxable income after the $19,125 head-of-household standard deduction, which exceeds the $15,892 AGI and zeroes the base."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $18,150 standard deduction, the phased senior deduction of $1,123, and the 2026 brackets. It never took the OBBBA $1,000 charitable deduction for non-itemizers, even though it chose the standard deduction and had $15,647 of cash gifts. Its taxable income of $137,003 is $1,000 too high, which adds 24% x $1,000 = $240 to the tax."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a made-up standard deduction of $20,550 plus $3,200, applied 2023-era brackets (10% to $11,000, 22% to $95,375), and at first included all $5,742 of Social Security. It also omitted the $1,123.45 phased senior deduction and the $1,000 non-itemizer charitable deduction. It then submitted $14,829, which contradicts its own bracket result of about $25,392."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It took the full $6,000 OBBBA senior deduction as 'phaseout-free' and ignored the 6% reduction on MAGI above $75,000, which cuts it to $1,123.45 at $156,276 MAGI. It also omitted the $1,000 non-itemizer charitable deduction and used approximate brackets. It then replaced its own $24,431 bracket result with an unsupported $13,802."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It named the senior bonus deduction but never subtracted it, left out the $1,000 non-itemizer charitable deduction, and inflated the standard deduction to $18,600 instead of $18,150. That left taxable income at $137,676 instead of $136,002.41. It also used a guessed top of $107,500 for the 22% bracket instead of $105,700."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted $1,181 of auto loan interest, but the OBBBA vehicle-loan interest deduction is fully phased out at this MAGI ($200 per $1,000 above $100,000). It also used about $17,750 of total deductions instead of $18,150 standard plus $1,123.45 senior plus the $1,000 non-itemizer charitable deduction. Its $24,196 is not even the 2026 bracket tax on its own $138,526 taxable income, which is about $25,844."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected 2026 parameters by inflating 2025 figures, giving a $17,700 standard deduction and a 22% bracket ending at $106,150. The actual 2026 figures are $16,100 plus $2,050 and $105,700. It also omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction, so its taxable income of $138,576 is about $2,574 too high."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It guessed a partial senior deduction of about $4,000 instead of computing $6,000 - 6% x ($156,276 - $75,000) = $1,123.45, and it omitted the $1,000 non-itemizer charitable deduction. It then cut its own $24,788 bracket result to an unsupported $18,033 by vaguely 'reconciling deduction phase-outs'."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got the $18,150 standard deduction, the $1,123 phased senior deduction, and the 2026 brackets right. It omitted the OBBBA $1,000 cash charitable deduction that standard-deduction filers take from 2026, even though it listed $15,647 of charity. Its taxable income of $137,002 is $1,000 high, which adds $240 of tax at 24%."
-us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,500 base standard deduction instead of $16,100 and made-up brackets (10% on $12,075, 22% on $55,700) instead of the 2026 schedule of $12,400, $50,400 and $105,700. It also omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction, so its taxable income is $138,725.70 instead of $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law, with a $9,934 standard deduction, a $5,293 personal exemption, and 15/25/28% brackets. OBBBA made the TCJA standard deduction, the zero personal exemption, and the 10/12/22/24% brackets permanent. The correct path is the $18,150 standard deduction plus the $1,123.45 senior deduction plus the $1,000 non-itemizer charitable deduction, giving $136,002.41 taxable income taxed at 2026 rates."
-us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had expired: it itemized $16,484, subtracted a $5,400 personal exemption, and used 15/25/28% brackets. OBBBA kept the TCJA rate schedule and standard deduction permanent. It also submitted $28,095.43, which does not match its own computed $28,345.68."
-us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It explicitly used 'pre-TCJA brackets' with a $5,419 personal exemption and 15/25/28% rates. OBBBA made the TCJA rates and the zero exemption permanent, so the tax is the 2026 10/12/22/24% schedule applied to $136,002.41, after the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It stated that 2026 returns to pre-TCJA rules, so it itemized $16,484, took a $5,300 personal exemption, and used 15/25/28% brackets. OBBBA made the TCJA structure permanent, so the standard deduction plus the senior and non-itemizer charitable deductions ($20,273.45) apply, taxed at the 2026 10/12/22/24% schedule."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $5,742 of Social Security as income (gross $157,137) instead of the 85% cap of $4,880.71. It used a $16,800 deduction, omitting the $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its $17,296 is far below the 2026 bracket tax on its own $140,337 taxable income, which is about $26,279."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It itemized $16,484 and took a personal exemption under the expired-TCJA framework. OBBBA made the TCJA standard deduction, the zero personal exemption, and the 10/12/22/24% brackets permanent. The correct path taxes $136,002.41 at 2026 rates to get $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied 'post-TCJA expiration' rules with a $5,050 personal exemption and pre-2018 brackets. OBBBA kept the TCJA rates and standard deduction permanently, so the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions apply, with the 2026 24% bracket starting at $105,700."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation beyond 'standard deductions and tax brackets'. Its $19,304 corresponds to taxable income of about $111,300 under the 2026 schedule, roughly $24,700 below the correct $136,002.41. The correct figure is AGI $156,275.86 minus the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its number reflects an understated bracket computation."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and the age-65 add-on, omitting the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. Its $26,105.17 implies taxable income of about $139,600 at 2026 rates, roughly $3,600 above the correct $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It took only the standard deduction for a single filer 65 or older and omitted the $1,123.45 phased senior deduction and the $1,000 non-itemizer charitable deduction. Its $25,836.43 implies taxable income of about $138,490, roughly $2,490 above $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied '2026 pre-TCJA rules' with a reinstated $5,450 personal exemption and a 28% marginal bracket. OBBBA made the TCJA structure permanent, so taxable income is $136,002.41 after the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions, taxed at a 24% top marginal rate."
-us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It estimated a $15,400 base standard deduction instead of $16,100 and omitted both the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. That left taxable income at $138,825.70 instead of $136,002.41. It also taxed the result with projected brackets (22% to $106,150) instead of the actual 2026 schedule."
-us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the 2025 base standard deduction of $15,000 instead of the 2026 figure of $16,100 and omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $139,226 is $3,224 too high, which overstates the 24%-bracket tax."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no numbers. Its $16,348 corresponds to taxable income of about $98,300 under the 2026 schedule, about $37,700 below the correct $136,002.41, which is AGI of $156,275.86 minus the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and itemization offset the income and returned $0. Total deductions are only $20,273.45 against AGI of $156,275.86, leaving $136,002.41 taxable in the 24% bracket, and no nonrefundable credits apply."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"It declined to compute and submitted 0 as an explicit placeholder. The correct computation taxes $136,002.41 of taxable income at 2026 rates, giving $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a 'senior-enhanced standard deduction'. Its $24,379 is $859 below the reference, which at the 24% marginal rate means about $3,580 of excess deduction. That matches taking most of the unphased $6,000 senior deduction instead of the $1,123.45 left after the 6% phaseout above $75,000 MAGI."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction, the aged add-on, and the phased senior deduction correctly but omitted the OBBBA $1,000 cash charitable deduction for non-itemizers. Its $25,479 is exactly $240 (24% x $1,000) above the correct $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It counted all $5,742 of Social Security (gross $157,137) instead of the 85% cap of $4,880.71 and used a $17,000 deduction, omitting the $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its $21,842 is also far below the 2026 bracket tax on its own $140,137 taxable income, which is about $26,231."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-expiration' law, with a roughly $10,500 standard deduction, a $5,400 personal exemption, and 10/15/25/28% brackets. OBBBA made the TCJA standard deduction and 10/12/22/24% rates permanent, so the tax on $136,002.41 of taxable income is $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed 'TCJA-sunset law', itemizing $16,484 and taking a $5,300 personal exemption at 10/15/25/28% rates. OBBBA made the TCJA rules permanent, so the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions apply, taxed at the 2026 24% bracket."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly spotted the $1,000 non-itemizer charitable deduction but subtracted it above the line to get a $155,276 AGI. That lowered the phaseout MAGI and inflated the senior deduction to $1,183 instead of $1,123.45. Its main error is bracket arithmetic: its own $135,943 taxable income produces $25,224 at 2026 rates, but it reported $25,314."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a projected post-TCJA standard deduction of $9,650 and 10/15/25/28% brackets. OBBBA made the TCJA standard deduction and 10/12/22/24% rates permanent. It also skipped the personal exemption it implied and reported taxable income of $139,792, which does not equal its AGI minus its $16,484 of itemized deductions ($139,792 is exactly that)."
-us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It itemized $16,484 against a 'pre-TCJA standard deduction', subtracted a $5,200 personal exemption, and used 10/15/25/28% brackets. OBBBA made the TCJA structure permanent, so taxable income is $136,002.41 after the standard, senior and non-itemizer charitable deductions, taxed at the 2026 24% bracket."
-us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It projected a roughly $17,400 standard deduction instead of $18,150 and omitted the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $138,876 is $2,874 too high, and it used projected brackets (12% band ending at $49,700) instead of the 2026 schedule."
-us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It invented about $11,980 of mortgage interest from the loan balance, even though no interest was listed. It then wiped out an estimated $18,500 of tax with nonrefundable credits that do not exist for this childless filer. No nonrefundable credits apply, so the tax before refundable credits is the full $25,238.58 on $136,002.41 of taxable income."
-us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as $2,045 instead of the 85% cap of $4,880.71, which applies because provisional income far exceeds $34,000. It used a $15,700 standard deduction with no aged add-on and then itemized $16,524 including sub-floor medical. It never applied the $18,150 standard, $1,123.45 senior or $1,000 non-itemizer charitable deductions, and it used made-up brackets (22% to $111,875)."
-us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $30,253 corresponds to taxable income of about $156,900 at 2026 rates, which is essentially all of AGI with no standard, senior or non-itemizer charitable deduction subtracted. The $20,273.45 of deductions brings taxable income down to $136,002.41."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $18,150 standard deduction, the $1,123 phased senior deduction and the 2026 brackets. But it left out the OBBBA §170(p) $1,000 charitable deduction for non-itemizers, even though the filer gave $15,647 in cash. That put taxable income at $137,003 instead of $136,002, and the tax came out exactly $240 (24% of $1,000) too high."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a made-up $23,750 standard deduction and 2023 bracket thresholds. It also left out the phased $1,123 senior deduction and the $1,000 non-itemizer charitable deduction. Its own bracket math produced about $25,392, but it then reported $14,829 with no calculation behind it."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It understated taxable Social Security at about $4,300 instead of $4,880.71 and showed no bracket math. Correctly, taxable income is $136,002 after the $18,150 standard deduction, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, which gives $25,239 at 2026 rates. Its $13,900 matches taxable income of only about $87,000, so it under-taxed the income by roughly $49,000."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It claimed the full $6,000 OBBBA senior deduction 'phaseout-free'. That deduction actually shrinks by 6% of MAGI above $75,000, leaving $1,123.45. It also missed the $1,000 non-itemizer charitable deduction. Its own bracket math gave $24,431, but it reported $13,802 with no supporting arithmetic."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It mentioned the senior bonus deduction but never subtracted the phased $1,123. It also left out the $1,000 non-itemizer charitable deduction, overstated the standard deduction at $18,600 instead of $18,150, and set the top of the 22% bracket at $107,500 instead of $105,700. The result was taxable income of $137,676 against the correct $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the $1,181 of auto loan interest as a deduction. That deduction is fully phased out once MAGI exceeds $100,000 by this much. It also left out the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction and used a $17,750 standard deduction. Its stated taxable income of $138,526 gives about $25,844 at 2026 rates, not the $24,196 it reported."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected 2025 parameters forward by inflation: a $15,700 base plus $2,000 aged add-on, and a 22% bracket ending at $106,150. The actual OBBBA-reset 2026 figures are $16,100 plus $2,050, with the 22% bracket ending at $105,700. It also left out the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $138,576 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It guessed the phased senior deduction at about $4,000. The 6% phaseout above $75,000 MAGI actually leaves $1,123.45. It also missed the $1,000 non-itemizer charitable deduction. Its own bracket math gave $24,788, which it then cut to $18,033 with no computation."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got the standard deduction, the $1,123 phased senior deduction and the 2026 brackets right. It left out the OBBBA §170(p) $1,000 charitable deduction for non-itemizers, so taxable income was $137,002 instead of $136,002 and the tax came out $240 too high."
+us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,500 base standard deduction instead of $16,100 and made-up bracket widths ($12,075/$37,100/$55,700). It also left out both the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $138,726 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: a $5,293 personal exemption, a roughly $9,934 standard deduction and 15/25/28% brackets. OBBBA made the TCJA rates and the larger standard deduction permanent and kept the personal exemption at zero. It also missed the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, so it itemized when the $18,150 standard deduction was larger."
+us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied repealed pre-TCJA 2026 rules: a $5,400 personal exemption and 15/25/28% brackets. Under OBBBA there is no exemption, the standard deduction is $18,150 plus a $1,123 senior deduction and a $1,000 non-itemizer charitable deduction, and the 10/12/22/24% brackets apply. Its submitted $28,095.43 also differs from its own computed $28,345.68."
+us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It assumed TCJA expired: a $5,419 personal exemption, itemizing $16,484, and 10/15/25/28% brackets. OBBBA kept the 10/12/22/24% brackets and the $16,100 base standard deduction permanently. With the aged add-on, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, taxable income is $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It explicitly assumed 2026 returns to pre-TCJA rules, with a $5,300 personal exemption and 15/25/28% brackets. OBBBA made TCJA permanent, so the filer takes the $18,150 standard deduction, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, and is taxed at 10/12/22/24%."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It treated all $5,742 of Social Security as taxable, though only 85% ($4,880.71) is. It used a $16,800 standard deduction instead of $18,150 and left out the $1,123 senior deduction and the $1,000 charitable deduction. Its own taxable income of about $140,337 gives about $26,800 at 2026 rates, so the reported $17,296 is also an arithmetic understatement."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: itemizing $16,484 plus a personal exemption, taxed at 15/25/28%. OBBBA kept the TCJA brackets and standard deduction and eliminated the exemption. The correct deductions are the $18,150 standard deduction, the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It claimed a $5,050 personal exemption and post-TCJA-expiration brackets, even though OBBBA made the TCJA rate structure permanent. Under the actual 2026 rules, the $18,150 standard deduction, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction leave $136,002 taxed at 10/12/22/24%."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no deduction or bracket math. The correct path is $156,276 AGI less $18,150 standard, $1,123 senior and $1,000 non-itemizer charitable deductions, which leaves $136,002 and $25,239 of tax. Its $19,304 matches taxable income of only about $111,000, so roughly $25,000 of income went untaxed."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted only the standard deduction and the age-65 add-on. It left out the OBBBA senior deduction, phased down to $1,123.45, and the $1,000 non-itemizer charitable deduction. Its $26,105 matches taxable income of about $139,600 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only a standard deduction with the aged add-on, about $17,800. It left out the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction, which leaves taxable income near $138,500 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used 'pre-TCJA rules' with a reinstated $5,450 personal exemption and the 28% bracket. OBBBA permanently kept the zero exemption and the 10/12/22/24% brackets. The correct deductions are the $18,150 standard deduction, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction."
+us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a projected $15,400 base standard deduction instead of $16,100 and inflation-projected brackets ($12,250/$49,800/$106,150) instead of the 2026 figures ($12,400/$50,400/$105,700). It also left out the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $138,826 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2025 $15,000 base standard deduction instead of the 2026 $16,100. It also left out the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $139,226 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no deduction amounts or bracket math. Correctly, $156,276 of AGI less the $18,150 standard, $1,123 senior and $1,000 non-itemizer charitable deductions leaves $136,002, which is $25,239 at 2026 rates. Its $16,348 matches taxable income below $100,000, so roughly $38,000 of income went untaxed."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed deductions offset the income completely. The available deductions total only $20,273 ($18,150 standard, $1,123 senior, $1,000 non-itemizer charitable) against $156,276 of AGI, so $136,002 remains taxable and owes $25,239."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"It refused to compute and submitted 0 as a placeholder. The facts were enough: $156,276 AGI less $20,273 of deductions gives $136,002 of taxable income and $25,239 of tax at 2026 brackets."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Its $24,379 matches taxable income of about $132,400, roughly $3,600 below the correct $136,002. That is consistent with taking most of the $6,000 senior deduction without the 6%-of-MAGI-over-$75,000 phaseout, which cuts it to $1,123.45."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction, the aged add-on and the phased senior deduction correctly. It left out the OBBBA $1,000 charitable deduction for non-itemizers, so taxable income was $137,003 and the tax came out exactly $240 too high at $25,479."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It included all $5,742 of Social Security in gross income instead of the capped 85% ($4,880.71). It deducted only $17,000, leaving out the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction. Its own $140,137 of taxable income gives about $26,231 at 2026 rates, not the $21,842 it reported."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a projected TCJA-sunset regime: about a $10,500 standard deduction, a $5,400 personal exemption and 15/25/28% brackets. OBBBA made the TCJA brackets and $16,100 base standard deduction permanent. With the aged add-on, the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, taxable income is $136,002 at 10/12/22/24%."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly assumed 'TCJA-sunset law' with a $5,300 personal exemption and 10/15/25/28% brackets. OBBBA repealed the sunset, so the filer takes the $18,150 standard deduction plus the $1,123 senior and $1,000 non-itemizer charitable deductions under the 10/12/22/24% schedule."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly took the $1,000 non-itemizer charitable deduction, but treated it as reducing AGI. It then ran the senior deduction phaseout on $155,276 and got $1,183. The §170(p) deduction is taken below the line, so the 6% phaseout runs on $156,276 MAGI and leaves $1,123.45. Its own $135,943 of taxable income gives about $25,224 at 2026 rates, not the $25,314 it reported."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed a post-TCJA-expiration $9,650 standard deduction and 10/15/25/28% brackets. OBBBA made the TCJA schedule permanent, with a $16,100 base plus a $2,050 aged add-on. It also left out the $1,123 senior deduction and the $1,000 non-itemizer charitable deduction, so it itemized when the standard deduction was larger."
+us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied 'pre-expiration' 2026 rules: itemizing $16,484, a $5,200 personal exemption and 15/25/28% brackets. OBBBA kept the TCJA 10/12/22/24% brackets and a zero exemption, and the correct deductions are the $18,150 standard, $1,123 senior and $1,000 non-itemizer charitable deductions."
+us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a projected $17,400 total standard deduction instead of $18,150 and projected brackets instead of the 2026 $12,400/$50,400/$105,700 thresholds. It also left out the $1,123 phased senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $138,876 instead of $136,002."
+us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It made up about $11,980 of mortgage interest from the loan balance, even though no interest paid was listed and unlisted inputs are $0. It mis-stated AGI as $139,737. It then wiped out its own estimate of about $18,500 with nonrefundable credits that do not exist, reporting 0 when $136,002 of taxable income owes $25,239."
+us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as $2,045. Provisional income of $154,266 triggers the 85% cap, which is $4,880.71. It used a $15,700 standard deduction with no aged add-on and no senior deduction, then itemized $16,524, including medical expenses below the 7.5% floor and charitable gifts with no 0.5%-of-AGI floor. The correct choice is $18,150 standard plus $1,123 senior plus $1,000 charitable."
+us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $30,253 matches taxing roughly $157,000, essentially all gross income, at 2026 rates with no deductions. The correct return subtracts the $18,150 standard, $1,123 senior and $1,000 non-itemizer charitable deductions to reach $136,002 of taxable income."
us,scenario_055,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model named the correct rule — 6.2% employee Social Security plus 1.45% employee Medicare on the $83,635 of wages, i.e. 7.65% × $83,635 — but never executed the multiplication, submitting a round $6,400 instead of the $6,398.08 its own stated formula produces. The error is purely arithmetic execution: it rounded the FICA product to the nearest hundred dollars rather than computing $5,185.37 + $1,212.71."
us,scenario_055,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"The model identified the right base and rates (6.2% Social Security under the wage base plus 1.45% Medicare on $83,635, no Additional Medicare Tax, no Florida employee payroll tax), but mis-multiplied: 7.65% × $83,635 = $6,398.08, while it reported $6,437.70, an effective 7.697% rate that is $39.62 too high. Every structural judgment was right and only the arithmetic failed."
us,scenario_055,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's explanation states the correct derivation — wages of $83,635 below the wage base, 6.2% plus 1.45% employee-side, no Additional Medicare Tax, no Florida payroll tax — which yields $6,398.08, yet it submitted $10,534, an effective 12.6% of wages. It abandoned the 7.65% employee share it had just described and emitted a figure only reachable by applying combined employer-plus-employee-level rates to the $83,635 wage."
@@ -4758,143 +4845,147 @@ us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"T
us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract."
us,scenario_056,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the $400 threshold rule: $4,340 of self-employment income is above, not below, $400. Applying the 92.35% net-earnings adjustment and 15.3% combined rate yields $613.22."
us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $4,340 below the $400 self-employment-tax threshold. Because the threshold is exceeded, the 92.35% net-earnings base is taxed at 15.3%, producing $613.22."
-us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added $564 a month of mortgage interest (plus an SUA with no utility costs listed) to the $996 in property taxes, then capped the excess shelter deduction at $744 to get $159 a month. Only real-estate taxes count, so the excess shelter deduction is $395.49 and net income is $805. The $63 formula benefit is then raised to NJ's $95 minimum."
-us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated all $18,722 of mortgage interest plus property tax as shelter costs and took the capped $8,928 annual shelter deduction. Only the $11,949 in real-estate taxes counts, which gives a $395.49 monthly excess shelter deduction and a $63 formula benefit. NJ's $95 monthly minimum then applies."
-us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 in gross income exceeds the one-person limit, but that is 112% of FPL, under NJ's BBCE gross limit. It also treated the mortgage balance as a countable asset, although the home is excluded and BBCE waives the asset test. It never applied the shelter deduction or NJ's $95 minimum."
-us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75 a month in real-estate taxes and overstated net income at about $1,400. The correct net income is $805.03 after the $395.49 shelter deduction. The household is eligible, and its $63 formula benefit is raised to NJ's $95 minimum instead of $0."
-us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Denied benefits because of assets implied by home ownership. The home is an excluded resource, and NJ BBCE waives the asset test. It also never worked out the property-tax shelter deduction or applied NJ's $95 minimum benefit that an eligible one-person household receives."
-us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 in countable income exceeds the gross and net limits, but gross income is 112% of FPL (under the BBCE limit). After the standard, earned-income and $395.49 property-tax shelter deductions, net income is 61% of FPL. The eligible household gets NJ's $95 monthly minimum."
-us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Counted only $487 a month of income, dropping the $12,000 of financial assistance that SNAP counts as unearned income. It also added mortgage interest to shelter costs, which drove net income to $0 and produced the maximum allotment."
-us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75 in monthly real-estate taxes, which put net income at $1,216 instead of $805. Having found the household eligible, it still returned $0 rather than the minimum allotment, which is $95 a month in NJ."
-us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of FPL, but gross income of $1,486.54 a month is 112% of the $1,330 guideline and under NJ's BBCE limit. It also skipped all deductions, including the $395.49 property-tax shelter deduction, which lead to a $95 NJ minimum benefit."
-us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $12,000 of financial assistance from income, although SNAP counts it as unearned income ($1,000 a month). It also added mortgage interest to shelter costs, so net income came out near zero and the answer was the maximum allotment instead of the $95 minimum."
-us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,Applied only the earned-income and standard deductions and left out the $395.49 excess shelter deduction for real-estate taxes. It also ignored that an eligible one-person household in NJ receives at least the $95 state minimum.
-us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the ABAWD time limit to cut the benefit to zero. In PolicyEngine this household passes the work-requirement test, so eligibility continues, and the benefit is NJ's $95 monthly minimum."
-us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Included mortgage interest in shelter costs and took a capped $712 excess shelter deduction, which gave net income of $498 and $142 a month. Only real-estate taxes count, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"A $0 answer from 30% of net income exceeding the maximum is what you get by leaving out the $395.49 property-tax shelter deduction. With it, net income is $805 and the formula benefit is $63, which is raised to NJ's $95 minimum."
-us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built shelter costs from mortgage interest, property taxes and utilities that were never listed, and took a capped $672 shelter deduction. Only the $995.75 in property taxes counts, giving a $395.49 deduction, a $63 formula benefit and NJ's $95 minimum."
-us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Said gross income exceeds the SNAP limits, but $1,486.54 a month is 112% of FPL, well under NJ's BBCE gross limit. The eligible household gets NJ's $95 monthly minimum."
-us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum (capped) excess shelter deduction based on mortgage interest plus property taxes, which gave net income of $489. Mortgage interest does not count, so the deduction is $395.49 and net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted only self-employment income and alimony and left out the $1,000 a month of financial assistance that SNAP counts, then awarded the $292 maximum allotment. With that income counted, net income is $805 and the benefit is NJ's $95 minimum."
-us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no derivation. The household is eligible at 112% of FPL gross, and after the $395.49 property-tax shelter deduction the $63 formula benefit is raised to NJ's $95 monthly minimum."
-us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72 monthly benefit implies net income of about $500, which requires a capped shelter deduction that includes mortgage interest. Only property taxes count, so net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Awarded the full $292 maximum, which implies zero net income. That result requires dropping the $12,000 of financial assistance and/or counting mortgage interest as shelter cost. Correct net income is $805, which gives NJ's $95 minimum."
-us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Disqualified the head as an ABAWD with no work. The household passes PolicyEngine's work-requirement test, so it stays eligible and receives NJ's $95 monthly minimum benefit."
-us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08 of mortgage interest to $995.75 of property taxes and capped the excess shelter deduction at $733. Only property taxes count, so the deduction is $995.75 − $600.27 = $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 of financial assistance as welfare income, although SNAP counts it as unearned income. It also left out the property-tax shelter deduction entirely, which produced a $207 benefit instead of NJ's $95 minimum."
-us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to zero without calculating anything. The household is income-eligible at 112% of FPL, and after the $395.49 property-tax shelter deduction the $63 formula benefit is raised to NJ's $95 monthly minimum."
-us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Assumed zero without applying the SNAP formula. Gross income of $1,486.54 passes the BBCE test, and the deductions, including the $395.49 property-tax shelter deduction, give a $63 formula benefit, which is raised to NJ's $95 minimum."
-us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly found that the formula benefit is below the minimum, but applied the federal $23 minimum allotment. New Jersey's state-funded minimum SNAP benefit of $95 a month applies instead, giving $1,140 a year."
-us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,Concluded that net income is too high for any allotment by leaving out the $395.49 excess shelter deduction for real-estate taxes. It also ignored NJ's $95 minimum benefit for an eligible one-person household.
-us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Took the capped excess shelter deduction by including mortgage interest in shelter costs, which gave $159 a month. With only property taxes counted, the deduction is $395.49 and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Explicitly built the capped shelter deduction from listed mortgage interest plus property taxes. Mortgage interest is not a counted shelter cost here, so the deduction is the uncapped $395.49, net income is $805, and the benefit is NJ's $95 minimum."
-us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Applied the $8,928 annual shelter cap after adding mortgage interest to property taxes, which gave net income of $5,535. With only the $11,949 in property taxes counted, net income is $9,660 a year, and the $63 monthly formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,Found that 30% of net income exceeds the maximum by leaving out the $395.49 property-tax excess shelter deduction. It also skipped NJ's $95 minimum benefit for an eligible one-person household.
-us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest as a shelter cost and applied the capped deduction, which gave net income of $5,535 a year. Only real-estate taxes count, which gives a $395.49 monthly deduction and a $63 formula benefit that NJ's $95 minimum replaces."
-us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Called the household income- and asset-ineligible. Gross income is 112% of FPL, under NJ's BBCE limit, and BBCE waives the asset test, so the household is eligible and receives NJ's $95 minimum."
-us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Acknowledged that income is within limits but returned zero, ignoring the $995.75 a month in listed real-estate taxes, which support a $395.49 excess shelter deduction. It also ignored NJ's $95 minimum for an eligible household."
-us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Allowed no shelter deduction because rent and utilities were not listed. The $11,949 in real-estate taxes is a SNAP shelter cost and produces a $395.49 excess shelter deduction, so the benefit is NJ's $95 minimum."
-us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance out of countable income, which cut net income to $210. It also set the shelter deduction to $0 despite listed real-estate taxes. With correct income and the $395.49 shelter deduction, the benefit is NJ's $95 minimum."
-us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for real-estate taxes, which left net income at $13,479 a year. With the $395.49 monthly shelter deduction, net income is $805 a month, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied a capped $712 shelter deduction based on mortgage interest plus property taxes, which gave net income of $493. Only property taxes count, so the deduction is $395.49 and net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score against the $1,140 reference."
-us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Included $564 of mortgage interest in $1,560.17 of monthly shelter costs and capped the excess at $744. With only the $995.75 in real-estate taxes, the excess shelter deduction is $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Asserted that income exceeds one-person limits, but gross income is 112% of FPL (under NJ's BBCE limit) and net income after deductions is 61% of FPL. The eligible household receives NJ's $95 minimum."
-us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Capped the excess shelter deduction using $564 of mortgage interest plus $996 of property taxes, which gave net income of about $500. Only property taxes count, so net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
-us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 exceeds a limit it put at $19,000, which contradicts its own comparison. It also denied benefits without applying the $395.49 property-tax shelter deduction or NJ's $95 minimum."
-us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds 130% of FPL, but $1,486.54 a month is 112% of the $1,330 guideline. It also left out the shelter deduction, which brings net income to $805 and gives a benefit at NJ's $95 minimum."
+us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added a standard utility allowance even though no utility expenses were listed, and counted the $564/month mortgage interest as a shelter cost, so it hit the $744 excess-shelter cap and got a $159/month benefit. Shelter costs are only the $995.75/month property taxes with no SUA, so the deduction is $395.49, net income is $805, the formula gives $63, and NJ's $95 minimum sets the benefit at $1,140 a year."
+us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated the $6,773 home mortgage interest plus property tax ($18,722) as SNAP shelter costs and took the capped $8,928 excess-shelter deduction. Under the reference reading, mortgage interest is a tax-only input and shelter costs are just the $11,949 property tax. That gives a $395.49/month deduction, $805 net income and a $63 formula benefit, raised to NJ's $95/month minimum."
+us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 of gross income exceeds the one-person SNAP limit, but $1,486.54/month is only 1.12 times FPL, well under NJ's broad-based categorical eligibility gross limit. It also treated a $100,341 mortgage balance, which is a debt and not an asset, as disqualifying resources. The household is eligible, and NJ's $95 minimum gives $1,140."
+us,scenario_056,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no income or deduction computation. The household is eligible with $805 net income after the standard, earned-income and property-tax shelter deductions. The $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year."
+us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75/month property taxes and overstated net income at about $1,400/month, so it wrongly failed the net income test. Net income is $805 (61% FPL). An eligible one-person household gets at least NJ's $95/month minimum even when 30% of net income exceeds the maximum allotment."
+us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit by assuming high assets from home ownership and the financial assistance. The home is an excluded resource, no countable assets are listed, and NJ's broad-based categorical eligibility removes the asset test. The household is eligible and receives NJ's $95/month minimum because the formula gives only $63."
+us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 of income exceeds the gross and net limits. Gross income is 1.12 times FPL, under NJ's broad-based categorical eligibility limit, and net income after the property-tax shelter deduction is 0.61 times FPL. The $63 formula benefit is raised to NJ's $95/month minimum."
+us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 financial assistance and used only $487/month of income. SNAP counts it as unearned income, so gross income is $1,486.54/month. With the property-tax-only shelter deduction of $395.49, net income is $805, not $0, so the benefit is the $95 NJ minimum rather than the maximum allotment."
+us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household passes the net income test, then set the benefit to $0 because 30% of net income exceeded the maximum allotment. An eligible one-person household always gets the minimum benefit, which is $95/month in New Jersey, so the answer is $1,140. It also left out the $395.49 property-tax excess shelter deduction."
+us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of FPL. Gross income of $1,486.54/month is 1.12 times the $1,330 guideline and passes, and NJ uses broad-based categorical eligibility anyway. Net income is $805, the formula gives $63, and NJ's $95 minimum applies."
+us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $12,000 financial assistance, which SNAP counts as unearned income, and counted mortgage interest so shelter costs hit the cap. Correct net income is $805/month with only the $395.49 property-tax shelter deduction, so the formula gives $63 and NJ's $95 minimum applies, not the $298 maximum."
+us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Set the benefit to $0 because 30% of net income reached the maximum allotment. It left out the $395.49 property-tax excess shelter deduction and ignored that an eligible one-person household receives NJ's $95/month minimum benefit, which is $1,140 a year."
+us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the ABAWD three-month time limit and cut eligibility to zero months, but the household passes the SNAP work requirement test under PolicyEngine's rules, so no time limit applies. It is eligible with $805 net income and receives NJ's $95/month minimum."
+us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Counted the home mortgage interest as a shelter cost and took a capped $712 excess-shelter deduction, giving net income of $498 and $142/month. Under the reference reading, only the $995.75/month property tax is a shelter cost. The deduction is then $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Concluded that 30% of net income exceeds the maximum allotment and gave $0. It left out the $395.49 property-tax excess shelter deduction, which brings net income to $805 and the formula to $63, and it ignored NJ's $95/month minimum benefit for eligible households."
+us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built a capped $672 shelter deduction from mortgage interest, property taxes and utility expenses, but no utility expenses were listed. Shelter costs are only the $995.75/month property taxes, so the deduction is $395.49 and net income is $805, not $544. The $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds the SNAP threshold. $1,486.54/month is 1.12 times FPL, well under NJ's broad-based categorical eligibility gross limit. The household is eligible, and the $63 formula benefit is raised to NJ's $95/month minimum."
+us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum excess shelter deduction using mortgage interest plus property tax, which gives $489 net income and $158/month. Under the reference reading, mortgage interest is a tax-only input, so the shelter deduction is the uncapped $395.49 from property tax alone. Net income is $805 and the formula gives $63, raised to NJ's $95 minimum."
+us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted only self-employment and alimony income, dropped the $1,000/month financial assistance, and awarded the full $292 maximum. With the assistance counted as unearned income, net income is $805/month, and the benefit is NJ's $95 minimum."
+us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no supporting computation. The household is eligible with $805 net income after the standard, earned-income and property-tax shelter deductions. The $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year."
+us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72/month benefit implies net income of about $500, which requires a capped excess-shelter deduction built from mortgage interest plus property tax. Under the reference reading, only the $995.75/month property tax counts, giving a $395.49 deduction and $805 net income. The $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Awarded the full $292 maximum allotment, which implies zero net income. That is only possible if the $12,000 financial assistance is ignored and an outsized shelter deduction is taken. Correct net income is $805/month, so the benefit is NJ's $95 minimum."
+us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Disqualified the head as a non-working ABAWD, but the household passes the SNAP work requirement test under PolicyEngine's rules. It is eligible with $805 net income and receives NJ's $95/month minimum."
+us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08/month of mortgage interest to the $995.75 property tax and capped the excess shelter deduction at $733, giving $159.63/month. Under the reference reading, mortgage interest is a tax-only input, so the excess shelter deduction is $995.75 − $600.27 = $395.49. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 financial assistance as TANF-type welfare and left out the property-tax shelter deduction. SNAP counts the assistance as unearned income, so gross income is $1,486.54 and net income is $805. The formula gives $63, which NJ's $95 minimum replaces."
+us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 without computing income or deductions. The household passes the gross and net income tests ($805 net, 61% FPL), and the $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year."
+us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 by citing the alimony and self-employment income without running the benefit formula. Net income after the standard, earned-income and property-tax shelter deductions is $805, the formula gives $63, and NJ's $95 minimum applies."
+us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household eligible and the formula benefit below the minimum, but applied the federal $23 minimum for one-person households. New Jersey funds a $95/month state minimum benefit, so the answer is $1,140, not $276."
+us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Set the allotment to $0 because net income was too high. It left out the $395.49 property-tax excess shelter deduction, which brings net income to $805 and the formula to $63, and it ignored NJ's $95/month minimum for eligible households."
+us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess-shelter deduction by counting mortgage interest with property tax, giving $159/month. Under the reference reading, mortgage interest is a tax-only input and the shelter deduction is the uncapped $395.49 from property tax alone. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Built a capped shelter deduction from the listed mortgage interest and property taxes, giving $158/month. Under the reference reading, only the $995.75/month property tax is a shelter cost, so the deduction is $395.49 and net income is $805. The $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Took the capped $8,928 annual shelter deduction by counting mortgage interest with property tax. Under the reference reading, only the $11,949 property tax counts, so the excess shelter deduction is $395.49/month and annual net income is about $9,660. The formula gives $63/month, which NJ's $95 minimum replaces."
+us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,Gave $0 because 30% of net income exceeded the maximum allotment. It left out the $395.49 property-tax excess shelter deduction and ignored that an eligible one-person household receives NJ's $95/month minimum benefit.
+us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Explicitly counted mortgage interest as a shelter cost and applied the $8,928 cap. Under the reference reading, mortgage interest is a tax-only input and the shelter deduction is the uncapped $395.49/month from property tax alone. That gives a $63 formula benefit, raised to NJ's $95 minimum."
+us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible on income and assets. Gross income is 1.12 times FPL under NJ's broad-based categorical eligibility limit, no countable assets are listed, and the home is excluded. The household is eligible and receives NJ's $95/month minimum."
+us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Admitted income is within limits but still gave $0. The $995.75/month property tax is a countable shelter cost producing a $395.49 deduction, and an eligible one-person household receives at least NJ's $95/month minimum, which is $1,140."
+us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Treated the absence of rent and utilities as zero shelter costs, but the $11,949 real estate tax is a homeowner shelter cost. That gives a $395.49 excess shelter deduction and $805 net income. The $63 formula benefit is raised to NJ's $95/month minimum rather than $0."
+us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted only alimony and self-employment income, dropping the $12,000 financial assistance that SNAP counts as unearned income. It also set shelter costs to $0 despite the property taxes. Correct net income is $805/month, so the benefit is NJ's $95 minimum, not $229/month."
+us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Left out the property-tax excess shelter deduction and returned $0 once 30% of net income exceeded the maximum allotment. Even its own $1,123/month net income is under the net limit, so the household is eligible and receives NJ's $95/month minimum."
+us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Included mortgage interest in shelter costs and applied the $712 cap, giving $493 net income and $144/month. Under the reference reading, only the $995.75/month property tax is a shelter cost, so the deduction is $395.49 and net income is $805. The $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value or explanation, so there was no answer to score."
+us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Summed mortgage interest and real estate taxes into $1,560.17/month of shelter costs and capped the excess at $744. Under the reference reading, mortgage interest is a tax-only input, so shelter costs are $995.75 and the uncapped deduction is $395.49. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed income exceeds the one-person limits. Gross income is 1.12 times FPL, under NJ's broad-based categorical eligibility limit, and net income is 0.61 times FPL. The household is eligible and receives NJ's $95/month minimum."
+us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess shelter deduction using $564/month of mortgage interest plus $996 of property tax, giving net income of about $500. Under the reference reading, mortgage interest is a tax-only input and the deduction is the uncapped $395.49 from property tax. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum."
+us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Contradicted itself on the gross limit and then asserted that income exceeds the threshold. Gross income of $1,486.54/month passes, and after the property-tax shelter deduction net income is $805. The $63 formula benefit is raised to NJ's $95/month minimum."
+us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds 130% of FPL and net income exceeds the net limit. Gross income is 1.12 times FPL, and net income after the $395.49 property-tax excess shelter deduction is 0.61 times FPL. The household is eligible and receives NJ's $95/month minimum."
us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_056,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model built the correct NJ gross income base of $5,839 but never applied New Jersey's $10,000 minimum filing threshold for single filers, which exempts this household's entire liability and drives the tax to $0.00. It compounded this by inventing a ""$2,000 standard deduction"" that New Jersey's gross income tax does not provide, then taxed the residual $2,839 at 1.4% to reach $39.75."
us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model reproduced PolicyEngine's intermediate NJ main income tax exactly — $5,838.53 of gross income less the $1,000 regular exemption, taxed at the 1.4% first-bracket rate — and submitted that $67.75 as the final answer. It omitted the last step: New Jersey's minimum filing threshold exempts a single filer whose gross income is at or below $10,000, so the $67.74 preliminary liability is fully offset to $0.00."
us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model swept the $12,000 of financial assistance into NJ gross income to reach $17,839, which New Jersey does not tax, and then claimed a $1,428 charitable deduction that the New Jersey gross income tax does not allow at all. Its residual $3,462 taxed at 1.4% yields $48.47, and it never reached the dispositive rule — the $10,000 single-filer minimum filing threshold that zeroes the liability on the true $5,838.53 gross income base."
us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs. This is a contract failure rather than a substantive modeling error: the correct derivation is $5,838.53 of NJ gross income less the $1,000 regular exemption at 1.4% = $67.74, fully exempted by New Jersey's $10,000 minimum filing threshold to $0.00."
us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $12,000 of financial assistance as New Jersey gross income, inflating the base to $13,772.82 after the $1,000 exemption, when NJ gross income is only the $1,498.53 alimony plus $4,340.00 self-employment income. It then abandoned its own $192.82 arithmetic and submitted $299.43, a figure no step in its reasoning produces, and it never applied New Jersey's $10,000 single-filer minimum filing threshold, which exempts the entire $67.74 preliminary liability to $0.00."
-us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied the federal 25-year minimum age for childless workers to the NJ credit and concluded that NJ EITC = $0. It missed that New Jersey extends its 40% EITC to childless filers aged 18+ who fail only the federal age test, which the engine values at $265.60."
-us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model identified NJ's age-18 childless EITC and computed 40% × (7.65% × $4,033.39 = $308.55) = $123.42. The reference instead equals 40% of the full $664 childless maximum ($265.60), so the entire gap comes from the phase-in step."
-us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model claimed New Jersey has no state EITC or other refundable credit. NJ has a refundable EITC equal to 40% of the federal credit and extends it to childless filers aged 18+, so this household is eligible."
-us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used an under-21 cutoff for the childless NJ EITC. That was the old 2020 floor; New Jersey lowered the minimum age to 18, so the 20-year-old head qualifies and the engine awards $265.60."
-us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model ruled out NJ refundable credits because the head is 20, has no children, and has unearned income. Alimony is not investment income for the EITC test, and New Jersey's childless EITC covers filers aged 18+, so the $4,033 of net self-employment earnings qualifies for $265.60."
-us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,The model required federal EITC eligibility (age 25+) for the NJ EITC. It missed New Jersey's rule that childless filers aged 18–24 get the NJ credit as if they met the federal age test.
-us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC to net earnings of $4,033.39 and took 40% of the $308.55 phased-in credit, giving $123.42. The reference equals 40% of the $664 childless maximum ($265.60), so the difference is only the phase-in treatment."
-us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model stated that the NJEITC requires federal EITC eligibility and applied the federal 25–64 childless age range. New Jersey waives the federal age floor for childless filers aged 18+, so the 20-year-old head receives the NJ EITC ($265.60)."
-us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model assumed the head could be a dependent and applied the federal 25+ childless-worker age requirement to the NJ EIC. The head files as a non-dependent, and New Jersey's childless EITC starts at age 18, producing $265.60."
-us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model claimed the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65 or older, blind, or disabled, and the model also missed the NJ EITC for childless filers aged 18+, which is the only refundable credit here ($265.60)."
-us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model said the NJ credit requires federal EITC eligibility, which the 20-year-old head lacks. New Jersey extends its EITC to childless filers aged 18+ who fail only the federal age test, so the credit is $265.60, not $0."
-us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model computed the NJ EITC as 40% of a $0 federal EITC. It ignored New Jersey's separate pathway for childless filers aged 18–24, who get the credit computed as if federally age-eligible."
-us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The model found the age-18 NJ pathway but applied 7.65% to the gross $4,340 of self-employment income. EITC earned income is net of the deductible half of self-employment tax ($4,033.39), and the engine's credit is $265.60, not 0.40 × 0.0765 × $4,340 = $132.80."
-us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model tied the NJ EITC strictly to a $0 federal EITC because of the head's age. It missed that New Jersey allows childless filers aged 18+ to claim the credit, which yields $265.60."
-us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model required federal EITC qualification for the NJ credit. New Jersey separately covers childless filers aged 18–24 who fail only the federal age test, so the head receives $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model answered $0 without analysis and never applied New Jersey's EITC for childless filers aged 18+. The head's $4,033 of net self-employment earnings qualifies for a $265.60 credit on that pathway."
-us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model recognized NJ's age-18 childless EITC and took 40% of a phased-in federal-formula credit of about $309, giving $123. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is the phase-in step."
-us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model applied NJ's age-18 look-alike EITC to about $4,008 of earnings and took 40% of a $307 phased-in credit, giving $123. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap comes from phasing in the credit."
-us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,The model answered $0 with no derivation and missed New Jersey's EITC for childless filers aged 18+. The engine awards $265.60 on the head's self-employment earnings.
-us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model concluded that $0 federal EITC and no children mean $0 NJ credits. New Jersey's EITC covers childless filers aged 18+ regardless of the federal age test, so the credit is $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model applied an age-21 floor to NJ refundable credits. New Jersey lowered the childless EITC minimum age to 18, so the 20-year-old head qualifies for $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"The model required federal EITC qualification for the NJ EITC and only considered child-based credits. It missed New Jersey's age-18+ childless EITC, which the engine values at $265.60."
-us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model carried the federal 25–64 childless age limit over to the NJ EITC. New Jersey waives that limit for childless filers aged 18+, which gives $265.60."
-us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model stated that the NJ childless EITC requires age 21+, which is the superseded 2020 threshold. The current NJ floor is 18, so the 20-year-old head qualifies and the engine awards $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model found no refundable NJ credit and missed the NJ EITC for childless filers aged 18+. The head's net self-employment earnings of $4,033.39 qualify for $265.60 on that pathway."
-us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated the lack of children as ruling out every NJ refundable credit. New Jersey's EITC is available to childless filers aged 18+ with earned income, which yields $265.60 here."
-us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model required positive NJ income tax liability and dependents before any refundable credit could apply. The NJ EITC is refundable, needs no liability, and is open to childless filers aged 18+, so it pays $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model asserted that no NJ EITC applies. New Jersey extends its EITC to childless filers aged 18+ who fail only the federal age test, so the 20-year-old head receives $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model claimed the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65 or older, blind, or disabled, and the model missed the NJ childless EITC for filers aged 18+, which supplies the $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model found the age-18 NJ pathway but computed the federal-formula credit on gross self-employment income (7.65% × $4,340 = $332). EITC earned income is net of the deductible half of self-employment tax ($4,033.39), and the engine's credit is $265.60."
-us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC to earnings net of half the self-employment tax and took 40% of the $308.55 phased-in credit, giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is only the phase-in step."
-us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model concluded that the head does not qualify for the state EITC. New Jersey covers childless filers aged 18+, so the 20-year-old head with $4,033 of net self-employment earnings receives $265.60."
-us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model identified NJ's childless EITC for a 20-year-old and took 40% of the $308.55 phased-in federal-formula credit, giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is the phase-in step."
-us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC and took 40% of the credit phased in on net earned income ($308.55), giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the difference is only the phase-in treatment."
-us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,The model answered $0 with no analysis and missed New Jersey's EITC for childless filers aged 18+. The engine awards $265.60 on the head's self-employment earnings.
-us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model treated the NJ EITC as purely 40% of a $0 federal EITC. It ignored New Jersey's age-18 childless pathway, which produces $265.60."
-us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model concluded that no federal EITC means no NJ EITC. New Jersey computes the credit for childless filers aged 18–24 as if they met the federal age test, giving $265.60."
-us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model correctly excluded the $50 property tax credit but zeroed out the NJ EITC because the federal EITC is $0. New Jersey extends its EITC to childless filers aged 18+, so the 20-year-old head receives $265.60."
-us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model set the NJ EITC to 40% of a $0 federal EITC. It missed New Jersey's rule that childless filers aged 18+ qualify despite failing the federal age test, which yields $265.60."
-us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model treated federal EITC ineligibility as disqualifying for the NJ EITC. New Jersey separately covers childless filers aged 18+, so the head receives $265.60."
-us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there was no answer to score against the $265.60 NJ EITC."
-us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model applied the federal 25–64 childless age range and set the NJ EITC to 40% of $0. New Jersey waives the federal age floor for childless filers aged 18+, so the credit is $265.60."
-us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model ruled out NJ refundable credits based on household composition. It missed that the NJ EITC covers single childless filers aged 18+ with earned income, which pays $265.60 here."
-us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model treated the NJ EITC as purely a percentage of the $0 federal EITC for an age-ineligible childless filer. New Jersey extends the credit to childless filers aged 18+, giving $265.60."
-us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model required qualifying children or other characteristics for the NJ EITC. New Jersey grants its EITC to childless filers aged 18+ with earned income, and the head's $4,033 of net self-employment earnings qualifies for $265.60."
-us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model required children and positive NJ tax liability for any refundable credit. The NJ EITC is refundable without liability and covers childless filers aged 18+, which yields $265.60."
+us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Applied the federal age 25 to 64 requirement for childless filers to the NJ EITC. It missed that NJ extends its refundable EITC to childless filers aged 18 and over using the federal formula. On $4,033.39 of net SE earnings, the reference awards $265.60."
+us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Correctly found the NJ childless EITC for ages 18 and up and the $4,033.39 earned income (net of half SE tax). It then multiplied the $308.55 phase-in amount by 40% to get $123.42. The reference's $265.60 is 40% of the $664 2026 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Said New Jersey has no state EITC or other refundable credit. In fact NJ has a refundable EITC that childless filers aged 18 and up can claim, so it never computed the $265.60 credit on $4,033.39 of net self-employment earnings."
+us,scenario_056,state_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"Required NJ tax liability and qualifying children before any refundable credit could apply. The NJ EITC is refundable regardless of liability and is open to childless filers aged 18 and up, which yields $265.60 here."
+us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used an under-21 cutoff, which was NJ's tax-year 2020 rule. Since 2021 the NJ childless EITC age floor has been 18, so the 20-year-old head qualifies on $4,033.39 of net SE earnings for the reference's $265.60."
+us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Treated the head's unearned income and unmarried-partner status as disqualifying. Alimony and financial assistance do not count toward the investment-income limit. The $4,340 of SE income gives $4,033.39 of EITC earned income, which qualifies for the NJ childless EITC at age 18+ ($265.60)."
+us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Required federal EITC eligibility and applied the federal under-25 exclusion. NJ separately extends its EITC to childless filers aged 18 and up, which produces the reference's $265.60."
+us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"Correctly applied NJ's childless EITC for ages 18 and up with earned income of $4,033.39 and a $308.55 federal-formula phase-in amount, then took 40% to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Made NJ EITC depend on federal eligibility and applied the federal childless age 25 minimum. It missed NJ's extension to childless filers aged 18 and up, which gives the reference's $265.60."
+us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Guessed that the head is someone's dependent and applied the federal age 25+ rule for childless filers. The head is the tax unit head, and NJ's EITC for childless filers aged 18+ applies, producing $265.60."
+us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Counted the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65+, blind, or disabled. It also missed the refundable NJ EITC for childless filers aged 18+, which makes up the reference's $265.60."
+us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Required federal EITC eligibility, which the 20-year-old fails because of the age 25 floor. It missed that NJ grants its EITC to childless filers aged 18 and up using the federal formula, giving $265.60."
+us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Computed the NJ EITC only as a share of the actual federal EITC ($0). It skipped NJ's separate pathway for childless filers aged 18 and up, which uses a federal-formula credit on $4,033.39 of earnings and yields $265.60."
+us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Correctly used NJ's childless EITC for ages 18 and up, but phased it in on gross SE income of $4,340. EITC earned income is net SE earnings minus half of SE tax ($306.61), or $4,033.39. Its 40% × 7.65% shortcut also differs from the reference's $265.60, which is 40% of the $664 maximum and comes from capping the $308.55 phase-in amount."
+us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Tied NJ EITC to a federal EITC of $0, which it got from the age-25 childless rule. It never applied NJ's age-18 floor for childless filers, under which the reference credit is $265.60."
+us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Made the NJ EITC depend on federal EITC eligibility. NJ independently covers childless filers aged 18 and up using the federal formula, which gives the reference's $265.60."
+us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Said no NJ refundable credit applies without checking the NJ EITC. It missed that the 20-year-old childless head, with $4,033.39 of net SE earnings, qualifies for NJ's childless EITC for ages 18 and up ($265.60)."
+us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC on $4,033 of earnings and took 40% of the ~$309 federal-formula credit to get $123. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Correctly used NJ's EITC for childless filers aged 18 to 24 and took 40% of a ~$307 federal-formula phase-in amount to get $123. It used $4,008 instead of $4,033.39 of earned income. The reference's $265.60 is 40% of the $664 maximum, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Returned $0 without evaluating the NJ EITC. The childless 20-year-old head qualifies under NJ's age-18 floor on $4,033.39 of net SE earnings, which gives the reference's $265.60."
+us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Concluded NJ credits are zero because the federal EITC is $0 and there are no children. NJ extends its EITC to childless filers aged 18 and up regardless of federal age eligibility, which yields $265.60."
+us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Used an age-21 minimum, which was NJ's 2020 rule for childless filers. The floor has been 18 since tax year 2021, so the 20-year-old head qualifies for the reference's $265.60 NJ EITC."
+us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Required federal EITC eligibility and mixed in the NJ Child Tax Credit's under-6 child rule. It missed that NJ's EITC covers childless filers aged 18 and up, producing $265.60."
+us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"Applied the federal childless age range of 25 to 64 to the NJ EITC. NJ sets its own age floor of 18 for childless filers, so the head qualifies for the reference's $265.60."
+us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Said the NJ childless EITC requires age 21+, which is the outdated 2020 rule. The floor has been 18 since tax year 2021, so the 20-year-old head qualifies for $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Found no NJ refundable credit and never evaluated NJ's EITC for childless filers aged 18 and up. On $4,033.39 of net SE earnings, that credit is $265.60 in the reference."
+us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Required children or explicit low-income indicators. NJ's refundable EITC needs no children for filers aged 18+, and the head's $4,033.39 of net SE earnings yields $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Made refundable credits depend on positive NJ tax liability and dependents. The NJ EITC is refundable and open to childless filers aged 18 and up, giving the reference's $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Said no NJ EITC applies, missing NJ's eligibility rule for childless filers aged 18 and up. Under that rule, the 20-year-old head's $4,033.39 of net SE earnings yields $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Treated the $50 NJ property tax credit as refundable for a non-elderly, non-disabled homeowner. It is refundable only for filers who are 65+, blind, or disabled. It also missed the NJ childless EITC (ages 18 and up) that makes up the reference's $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Correctly used NJ's age-18 childless EITC, but computed a $332 federal-formula credit on gross SE income of $4,340. EITC earned income is net of the $306.61 half-SE-tax deduction, or $4,033.39. Taking 40% of the phase-in amount also differs from the reference's $265.60, which is 40% of the $664 maximum and comes from capping the phase-in amount."
+us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC on earnings net of half SE tax, then took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Said the head does not qualify for the state EITC, missing NJ's extension to childless filers aged 18 and up. With $4,033.39 of net SE earnings, that credit is $265.60."
+us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"Correctly found the 20-year-old childless head eligible for the NJ EITC and took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC to net earned income and took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum."
+us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,Said no state refundable credits apply without evaluating the NJ EITC. The childless 20-year-old head qualifies under NJ's age-18 floor for $265.60.
+us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Derived the NJ EITC only from the actual federal EITC ($0). NJ computes a federal-formula credit for childless filers aged 18 to 24 who fail the federal age test, which yields $265.60."
+us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,Assumed no federal EITC means no NJ EITC. NJ's separate pathway for childless filers aged 18 and up covers the 20-year-old head and produces $265.60.
+us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"Correctly ruled out the $50 property tax credit as non-refundable for this head, but tied the NJ EITC to a federal EITC of $0. It missed NJ's eligibility for childless filers aged 18 and up, which gives the reference's $265.60."
+us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Calculated the NJ EITC as 40% of an actual federal EITC of $0. It ignored NJ's age-18 floor for childless filers, under which the head gets the reference's $265.60."
+us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"Treated federal EITC ineligibility as ruling out the NJ EITC. NJ extends the credit to childless filers aged 18 and up, which yields $265.60 on $4,033.39 of net SE earnings."
+us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no answer to score against the reference's $265.60 NJ EITC."
+us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"Applied the federal age 25 to 64 range for childless filers and calculated the NJ EITC as 40% of $0. It missed NJ's extension to childless filers aged 18 and up, which produces $265.60."
+us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Ruled out NJ refundable credits based on household composition alone. A childless single adult aged 18+ qualifies for the NJ EITC, giving $265.60 here."
+us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Derived the NJ EITC from a federal EITC of $0 because the filer fails the age test. NJ separately covers childless filers aged 18 and up, which yields the reference's $265.60."
+us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Required qualifying children for NJ EITC eligibility. Filers aged 18+ with no children and earned income qualify, and the head's $4,033.39 of net SE earnings gives $265.60."
+us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"Required qualifying children and positive NJ tax liability. The NJ EITC is refundable and covers childless filers aged 18 and up, producing the reference's $265.60."
us,scenario_056,tanf,glm-5.3,llm_error,categorical_eligibility,False,"The model read the $12,000 `financial_assistance` input as if it were reported TANF receipt and echoed it back verbatim as the computed benefit, when the question asks what PolicyEngine computes for `tanf` from the household facts. It never applied TANF's categorical requirement that the assistance unit contain a dependent child — New Jersey's WorkFirst NJ TANF covers only units with a child under 18 (or 18 and in school full time), while childless adults fall to WFNJ General Assistance, which is not TANF — so this childless 20-year-old head yields $0. Its answer of exactly 12000 is arithmetically identical to the input line it copied, confirming no eligibility test or benefit computation was performed at all."
us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no `tanf` entry in its `outputs` object and no explanation, so the required key was absent rather than substantively wrong. The correct derivation is that WorkFirst NJ TANF requires an assistance unit containing a dependent child, and this single 20-year-old head with zero children fails that categorical test, producing $0."
-us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly placed earned income of about $9,604 on the childless-EITC plateau. It then used a guessed maximum of about $659 and cut it to $636 for 'lookup-table rounding'. On the plateau the credit equals the 2026 maximum of $664, and the EITC table does not reduce a plateau credit."
-us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It made refundable credits depend on having positive income tax liability, and it claimed the income exceeds EITC thresholds. The EITC is refundable whatever the liability is, and about $9,665 of earned income is on the childless plateau, which gives $664."
-us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning correctly found a joint-return age pass (head 43) and a plateau credit of about $649, but it submitted an unrelated $397 that no step of its reasoning supports. The $649 it did reason to is also the 2025 maximum rather than the 2026 maximum of $664."
-us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It placed earned income of about $9,665 in the childless-EITC phase-out region and arrived at a small $218 credit. The joint phase-out does not begin until about $18,000 of AGI, so the household is on the plateau and gets the full 2026 maximum of $664."
-us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It assumed a 2026 childless maximum of about $690 and then reduced it to $566 as if the household were partly phased out. Earned income of $10,400 gross (about $9,665 net) sits on the plateau below the joint phase-out start, so the credit is exactly the $664 maximum."
-us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Its reasoning estimated a childless maximum of about $649 and a credit of about $600. It then submitted $967, which is above any childless-EITC maximum, apparently from treating both spouses as separate workers. A joint return gets one childless credit, capped at the 2026 maximum of $664."
-us,scenario_057,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly found the phase-in fully complete and AGI below the phase-out start, but it capped the credit at an assumed 2026 maximum of $650. The 2026 statutory maximum for no qualifying children is $664."
-us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used the 2025 maximum of $649 as the 2026 value. The inflation-adjusted 2026 maximum is $664.
-us,scenario_057,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"It required both spouses to meet the 25-64 age test and denied the credit because the spouse is 20. Under IRC 32(c)(1)(A)(ii)(II), a joint return passes if either the taxpayer or the spouse is 25-64, and the 43-year-old head does, so the $664 plateau credit applies."
-us,scenario_057,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $666. The statutory 2026 maximum for no qualifying children is $664.
-us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,It correctly identified the plateau but inflated the wrong base: it treated $600 as the 2024 maximum (the actual figure was $632) and projected $630. The 2026 childless maximum is $664.
-us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It denied the EITC for lack of a qualifying child and for failing an earned-income threshold. The no-qualifying-child EITC is available to a joint filer with one spouse aged 25-64, and about $9,665 of net SE earnings puts the household on the $664 plateau."
-us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $666 instead of the statutory $664.
-us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,It recognized childless-EITC eligibility but understated the 2026 maximum as $620. On the plateau the credit equals the full 2026 maximum of $664.
-us,scenario_057,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It treated self-employment income and small disability benefits as disqualifying for the EITC. Net SE earnings are earned income for EITC purposes, disability benefits do not bar the credit, and the head's age of 43 satisfies the joint childless age test, which yields $664."
-us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,It correctly identified the childless joint EITC but used a guessed maximum of $660 instead of the 2026 statutory maximum of $664.
-us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly identified the childless joint EITC but submitted $649, which is the 2025 maximum. The 2026 inflation-adjusted maximum is $664."
-us,scenario_057,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly computed earned income of $9,665.26 and passed the age test through the head, but it capped the credit at the 2025 maximum of $649 instead of the 2026 maximum of $664."
-us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used invented childless-EITC parameters ($1,231 and $1,283 maxima, a $9,580 breakpoint) and a wrong earned-income figure of $8,320. The 2026 no-qualifying-child maximum is $664, and the household's about $9,665 of earned income sits on that plateau."
-us,scenario_057,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the 7.65% phase-in rate to the full $10,400 with no cap and got $795.60. The phase-in stops at the earned-income amount (about $8,700 in 2026), which limits the credit to the $664 maximum. It also used gross SE income instead of net earnings after the half-SE-tax deduction."
-us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It ruled out the EITC because the household has no wages or qualifying children. Net self-employment earnings count as earned income for the EITC, and the childless credit is available, so the household gets the $664 plateau amount."
-us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted that no refundable credit is triggered and never evaluated the no-qualifying-child EITC. The head (43) meets the age test, and about $9,665 of net SE earnings puts the household on the $664 plateau."
-us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly placed the household on the childless plateau through the head's age, but it used a projected 2026 maximum of $667 instead of the statutory $664."
-us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,It correctly identified the plateau childless EITC but used the 2025 maximum of $649 instead of the 2026 maximum of $664.
-us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It held that the 20-year-old spouse makes the joint return fail the childless-EITC age test. On a joint return, only one spouse has to be 25-64, and the 43-year-old head qualifies, so the $664 credit applies."
-us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It claimed income phase-outs reduce the EITC to zero. AGI of about $9,665 is roughly half the joint childless phase-out start of about $18,000, so no phase-out applies and the full $664 is allowed."
-us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly applied the age test through the head and placed the household on the plateau, but it used a projected 2026 maximum of $667 instead of the statutory $664."
-us,scenario_057,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly passed the joint age test and placed the household below the phase-out, but it used a 2026 maximum of $666 instead of the statutory $664."
-us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $648 instead of the statutory $664.
-us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"It denied the childless EITC because the spouse is 20. On a joint return the age test is met when either spouse is 25-64, and the 43-year-old head satisfies it, which yields the $664 plateau credit."
-us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"It required both joint filers to be at least 25 for the childless EITC and zeroed the credit over the 20-year-old spouse. The statute requires only the taxpayer or the spouse to be 25-64, and the 43-year-old head qualifies, so the credit is $664."
-us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It claimed the head's $10,400 of SSTB self-employment income exceeds an EITC threshold. SSTB status has no bearing on the EITC, and about $9,665 of net earnings sits on the childless plateau below the joint phase-out, which gives $664."
-us,scenario_057,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly placed about $9,665 of earned income on the plateau but used a wrong 2025 maximum of $611 (the actual figure was $649) and indexed it to $625. The 2026 childless maximum is $664."
-us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed income falls below an EITC threshold and that disability benefits reduce eligibility. The childless EITC phases in from the first dollar of earned income, disability benefits have no effect on it, and about $9,665 of net SE earnings yields the $664 maximum."
-us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It denied the EITC because there is no wage income and no qualifying children. Net self-employment earnings count as EITC earned income, and the childless credit applies through the 43-year-old head, which gives $664."
-us,scenario_057,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that income exceeds Louisiana's Medicaid thresholds, but the household's MAGI is 0.45 x FPL — roughly one third of Louisiana's 138% FPL expansion-adult limit — so no income test excludes the head. It also never applied the SSI-recipient pathway: the head's blindness and disability with $10,400 of self-employment income yields $7,231 of SSI after the $20 general, $65 earned-income, and one-half exclusions, and Louisiana is a 1634 state where SSI receipt confers automatic Medicaid regardless of income."
-us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the disability flags and the $14 of disability benefits as the only disability-linked inputs and concluded no eligibility determinant was present, never computing SSI from those facts. Blindness plus disability with a non-dependent 43-year-old whose $10,400 self-employment income nets to low countable income under the SSI earned-income exclusions produces $7,231 of SSI, and SSI receipt is itself the Medicaid category (SSI_RECIPIENT) that grants automatic eligibility in Louisiana."
-us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model enumerated only the pregnancy, parent/caretaker, and aged-65+ pathways and declared none applied, omitting both the SSI-recipient categorical pathway and the fact that a 43-year-old childless adult is exactly who Louisiana's ACA expansion group covers. Its 'at this income level' qualifier inverts the arithmetic: MAGI is 0.45 x FPL, well under the 138% FPL expansion ceiling, and the head's $7,231 of SSI independently triggers automatic Medicaid under Louisiana's 1634 agreement."
+us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly placed earned income of about $9,604 on the plateau, below the phase-out. It then cut its own ~$659 maximum to $636 with an invented 'lookup-table rounding' adjustment. On the plateau the credit equals the 2026 statutory maximum of $664, with no table reduction."
+us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model assumed that zero taxable income means no refundable credits, and claimed income exceeded the EITC thresholds. The EITC is refundable whatever the tax liability. AGI of about $9,665 is far below the roughly $18,000 joint phase-out start, so the full $664 childless credit applies."
+us,scenario_057,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model ruled out the childless EITC with muddled reasoning about qualifying status. A joint filer with no children qualifies when either spouse is 25–64, and the 43-year-old head meets that test. Self-employment earnings of about $9,665 put the household on the $664 plateau."
+us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model's reasoning ended on a plateau maximum of about $649, a 2025-level figure, yet it submitted $397, which nothing in its work supports. The correct result is the 2026 childless maximum of $664, because earned income of about $9,604 exceeds the phase-in ceiling and AGI is below the joint phase-out."
+us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model put AGI of about $9,665 in the childless EITC phase-out range. The 2026 joint phase-out does not start until about $18,000, so the household is on the plateau and gets the full $664, not $218."
+us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model guessed a ~$690 maximum and then applied a partial phase-out to reach $566, even though its own reasoning notes that phase-out starts above ~$18,000 for joint filers. Earned income of about $9,665 is on the plateau, so the credit is the 2026 maximum of $664."
+us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model reasoned its way to about $600, then submitted $967 after treating both spouses as workers. The childless EITC is a single credit per joint return, capped at the 2026 maximum of $664, and the household is on the plateau, so the credit is exactly $664."
+us,scenario_057,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model capped the credit at $650, essentially the 2025 maximum of $649. The 2026 inflation-adjusted maximum for filers with no qualifying children is $664, and earned income of about $8,870–$9,665 is past the phase-in, so the credit is $664."
+us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the 2025 childless EITC maximum of $649 without indexing it to 2026. Its plateau placement is correct, but the 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"The model required both spouses to be 25–64 for the childless EITC. IRC §32(c)(1)(A)(ii)(II) requires only the taxpayer or the spouse to meet the age range, and the 43-year-old head does, so the household gets the $664 plateau credit."
+us,scenario_057,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The plateau placement is correct, but the model projected the 2026 childless maximum at $666. The published 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model indexed a wrong 2024 base of $600 (the actual figure was $632) and arrived at $630. The 2026 childless EITC maximum for a household on the plateau is $664.
+us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said the household fails the qualifying-child or earned-income requirements. The childless EITC needs no qualifying child, the 43-year-old head meets the age test, and about $9,665 of self-employment earnings puts the household on the $664 plateau."
+us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly put the household on the childless EITC plateau but overstated the 2026 maximum as $666. The indexed 2026 maximum is $664.
+us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly treated the household as eligible but understated the 2026 childless maximum at $620. Earned income is past the phase-in and below the phase-out, so the credit is the full 2026 maximum of $664."
+us,scenario_057,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model claimed self-employment income and disability benefits disqualify the household from the EITC. Net self-employment earnings count as earned income, and the $38 of disability benefits has no effect on eligibility, so the childless EITC is the full $664."
+us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The plateau placement is correct, but the model used $660 as the 2026 childless maximum. The indexed 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model applied the 2025 childless EITC maximum of $649 instead of the 2026 figure of $664 for a household on the plateau.
+us,scenario_057,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly computed earned income of $9,665 and applied the age test correctly, but capped the credit at the 2025 maximum of $649. The 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an invented $1,231/$1,283 maximum and a wrong $8,320 earned-income figure for the childless EITC. The 2026 childless maximum is $664, and earned income of about $9,665 from self-employment puts the household on the plateau at exactly that amount."
+us,scenario_057,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model multiplied the full $10,400 by the 7.65% phase-in rate and never capped the result at the maximum credit. The phase-in ends at about $8,680 of earned income, so the credit tops out at the 2026 maximum of $664."
+us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated wages as the only earned income that supports an EITC. Net self-employment earnings (about $9,665) are earned income for the EITC, which puts the childless household on the $664 plateau."
+us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model never evaluated the childless EITC. The household's self-employment earnings, the 43-year-old head's age, and AGI below the joint phase-out produce the full $664 credit."
+us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The plateau placement is correct, but the model projected the 2026 childless maximum at $667. The indexed 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,The model used the 2025 childless EITC maximum of $649 instead of the 2026 maximum of $664.
+us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model held that the 20-year-old spouse makes the joint return fail the childless EITC age test. Only one spouse on a joint return has to be 25–64, and the 43-year-old head satisfies that, so the credit is $664."
+us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model claimed income phase-outs eliminate the EITC. AGI of about $9,665 is well below the roughly $18,000 joint phase-out start for 2026, so the full $664 childless credit applies."
+us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The age test and plateau placement are correct, but the model projected the 2026 childless maximum at $667. The indexed 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The age test and plateau placement are correct, but the model overstated the 2026 childless maximum as $666. The indexed 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,The model used a 2025-level maximum of about $648 instead of the 2026 childless EITC maximum of $664 for a household on the plateau.
+us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"The model disqualified the household because the spouse is 20. For a joint return, the childless EITC age test is met when either spouse is 25–64, and the 43-year-old head qualifies, so the credit is $664."
+us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"The model required both spouses to be at least 25. The childless EITC rule needs only the taxpayer or the spouse to be in the 25–64 range, and the head at 43 meets it, which yields the $664 plateau credit."
+us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model invented an EITC income threshold tied to SSTB status. The SSTB label matters only for the QBI deduction, and earned income of about $9,665 is on the childless plateau, so the credit is $664."
+us,scenario_057,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model used a wrong 2025 base of $611 (the actual figure was $649) and indexed it to $625. The 2026 childless maximum for a household on the plateau is $664.
+us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model claimed income was below a threshold and that disability benefits reduce EITC eligibility. Neither is an EITC rule: $9,665 of self-employment earnings exceeds the phase-in ceiling, and the $38 of disability benefits is irrelevant, so the credit is $664."
+us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the absence of wages and children as ruling out the EITC. Net self-employment earnings are earned income, and childless joint filers with one spouse aged 25–64 qualify, giving the $664 plateau credit."
+us,scenario_057,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said the head's self-employment income plus disability benefits exceeds 138% FPL. Net self-employment earnings of about $9,700 for a two-person household come to only 0.45× FPL, well under the expansion limit. The model also missed that the blind, disabled head receives SSI (about $7,231), which gives automatic Medicaid in the SSI_RECIPIENT category."
+us,scenario_057,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model said the head's income exceeds Louisiana's Medicaid limits. Household MAGI is 0.45× FPL, far below the 138% expansion limit. The model also ignored the head's SSI receipt (about $7,231 as a blind and disabled person), which makes Medicaid automatic through the SSI_RECIPIENT category."
+us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model answered 0 because it judged the disability flag and small disability benefits insufficient on their own. It never worked out that a blind and disabled head with about $10,400 of self-employment income and $912 in assets qualifies for SSI (about $7,231). SSI receipt confers automatic Louisiana Medicaid through the SSI_RECIPIENT category."
+us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the head only as a non-parent, non-elderly adult and ruled out expansion 'at this income level'. Louisiana is a Medicaid expansion state, and household MAGI of 0.45× FPL is well under 138%, so the head qualifies on income. The model also ignored the head's blindness and disability, which make the head an SSI recipient (about $7,231) with automatic Medicaid in the SSI_RECIPIENT category."
us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated being disabled as enough for Medicare at any age. Under 65, Medicare requires 24 months of SSDI entitlement (or ESRD/ALS). The head is 43 and has zero months of Social Security disability receipt, so they fail both the age-65 test and the disability test."
us,scenario_057,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model read the $14 of 'disability benefits' as SSDI entitlement. That input is generic, non-Social Security disability income. The head has zero months of Social Security disability receipt, so they fail the 24-month SSDI requirement, and at 43 they are below the age-65 threshold."
us,scenario_057,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated the rule requiring 24 months of SSDI, then dropped it. It concluded that anyone disabled and 'receiving disability benefits' qualifies, and treated $14 of generic disability benefits as Social Security disability entitlement. No SSDI or months of SSDI receipt are listed, so the 24-month test fails and the 43-year-old head is not eligible."
@@ -4913,52 +5004,53 @@ us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out
us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model computed the correct $1,469.46 from $9,604.40 × 15.3%, then added roughly $82.60 of 0.9% Additional Medicare Tax by treating $400 as the AMT threshold; the AMT threshold is $250,000 of combined earnings for joint filers, and the prompt explicitly excludes Additional Medicare Tax from this output. Its arithmetic is also self-contradictory — the stated components sum to $1,552.06, not the $1,531.79 submitted — where the correct answer is the unadjusted $1,469.47."
us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model fabricated a ""$12,000 threshold for self-employment tax liability"" and zeroed the tax; no such filing floor exists, and the only threshold in §1402(b) is $400 of net earnings, which $9,604.40 clears. Applying the 15.3% rate to that base gives $1,469.47."
us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model derived the correct liability — 0.153 × $9,604.40 = $1,469.47 — and then reported half of it, confusing the §164(f) above-the-line deduction for one-half of SE tax with the tax owed; that deduction reduces AGI and never reduces the §1401 liability itself. Its submitted $734.76 is not even the correct half ($734.74), and the requested output is the full $1,469.47."
-us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Built gross income from only the $866.67/month of self-employment earnings and $3.17 of disability benefits and never counted the head's $602.58/month SSI as unearned income. That left net income at about $485 instead of $1,090.08. With SSI counted, the $546 allotment minus a $327 contribution gives $219/month, then $232.74 from October, not the ~$380/month it annualized."
-us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,099/month, the couple rate that treats both spouses as SSI recipients. The household's SSI is the head's individual payment of $602.58/month ($994 FBR minus $391.42 countable). The inflated SSI pushed net income to $1,586 instead of $1,090.08, which cut the benefit to ~$70/month instead of $219/month (January–September) and $232.74/month (October–December)."
-us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Asserted a $0 benefit because income was 'slightly exceeding the net income limit' but never computed net income, and it wrongly shrank self-employment income by the 92.35% SE-tax factor. Net income is $1,090.08, which is 62% of the $1,762.50 guideline, so the household passes and receives $546 − $327 = $219/month, rising to $232.74 in October–December."
-us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of countable income and subtracted a monthly $219 standard deduction from an annual income figure. It then discarded its own $3,990 result and submitted $6,240 with no derivation. The correct monthly net income including SSI is $1,090.08, which yields $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of gross income and inflated the $38 annual disability benefits to $456 by treating them as monthly. It then added about $1,200 for unspecified 'disability/medical deductions', even though the $40/year of OTC expenses never exceeds the $35/month medical threshold. Including SSI, net income is $1,090.08 and the benefit is $219/month, then $232.74."
-us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Asserted net income near zero and used a two-person maximum of about $304, but the FY2026 two-person maximum is $546. The $866.67/month of self-employment earnings plus the head's $602.58/month SSI leave $1,090.08 of net income after the 20% earned-income and $209 standard deductions. That produces a $327 contribution and a $219/month benefit, not a near-maximum allotment."
-us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of $1,098.58/month, the couple rate with both spouses as SSI recipients, instead of the head's individual SSI of $602.58/month ($994 FBR minus $391.42 countable income). That overstated net income at $1,586 instead of $1,090.08 and produced $70/month instead of $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Excluded the head's $602.58/month SSI from SNAP income. It also cut self-employment income with an invented 40% cost-of-production deduction, counting $6,240 instead of the full $10,400 ($866.67/month). The correct gross income of $1,472.42 and net income of $1,090.08 give $219/month, not its $473/month."
-us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Reduced self-employment income by an income-tax-style SE-tax adjustment, treated the $38 annual disability benefits as monthly, and omitted the head's $602.58/month SSI. It then reported $297/month without any arithmetic connecting that figure to its stated ~$2,000 net income. The full $866.67/month of earnings plus $602.58 of SSI and $3.17 of benefits gives net income of $1,090.08 and a $219/month benefit."
-us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,099/month at the couple rate instead of the head's individual SSI of $602.58/month. That raised net income to $1,586 instead of $1,090.08 and left a ~$70/month benefit instead of $546 − $327 = $219/month, plus $232.74/month for October–December."
-us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Counted only $869.84/month of earnings and disability benefits and omitted the head's $602.58/month SSI. Net income came to $488.51 instead of $1,090.08, and the benefit to ~$389/month instead of $219 (January–September) and $232.74 (October–December). It also used a $536 maximum instead of the FY2026 two-person $546."
-us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Built gross income of $870/month from earnings and disability benefits alone and never added the head's $602.58/month SSI. It also used a $204 standard deduction instead of $209 and a $536 maximum instead of $546. Its net income of $489.34 compares with the correct $1,090.08, which yields $219/month rather than $389.20."
-us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted SSI of $1,067.17/month at the couple rate, as if both spouses were recipients, and treated the $38 annual disability benefits as monthly. The household's SSI is only the head's $602.58/month individual payment. The overstated net income of $1,594.50 (vs $1,090.08) cut the benefit to $57.65/month instead of $219."
-us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Omitted the head's $602.58/month SSI from gross income and used a $204 standard deduction instead of $209. That gave net income of $492.50 instead of $1,090.08 and a $398/month benefit instead of $546 − $327 = $219/month (then $232.74 in October–December)."
-us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted $13,122.96/year of SSI at the couple rate instead of the head's individual $602.58/month ($7,230.96/year). It also used a $198/month standard deduction instead of $209, and claimed a $60 medical deduction even though the $40/year of OTC expenses never exceeds the $35/month threshold. The correct net income of $1,090.08/month gives $219/month, not about $74."
-us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Declared the household ineligible with no computation. Net income of $1,090.08 is 62% of the poverty guideline, gross income is 84%, and the $912 of assets is far under the limit. The household therefore receives $546 − $327 = $219/month, then $232.74, totaling $2,669.22."
-us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Included SSI at the couple rate of about $1,099/month rather than the head's individual $602.58/month. Its $76/month benefit reflects net income near $1,567 instead of $1,090.08; the correct $327 contribution against the $546 maximum leaves $219/month."
-us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729/year of unearned income, which is couple-rate SSI for both spouses plus $38 of disability benefits. The household's SSI is only the head's $602.58/month individual payment. The excess SSI drove the benefit down to about $65/month instead of $219/month (January–September) and $232.74/month (October–December)."
-us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted a zero benefit without applying any test. The household passes the gross (84% of FPL), net (62% of FPL), and asset ($912) tests. Net income of $1,090.08 yields $546 − $327 = $219/month, then $232.74 from October."
-us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave $268/month with no derivation; against the $546 maximum, that implies net income of about $927. The correct net income is $1,090.08: add the head's $602.58/month SSI to $866.67 of earnings and $3.17 of benefits, then subtract the 20% earned-income and $209 standard deductions. That gives $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave $282/month with no derivation; against the $546 maximum, that implies net income of about $880 instead of $1,090.08. It undercounted income by not including the head's full $602.58/month SSI alongside $866.67 of earnings and $3.17 of benefits. The correct $327 contribution leaves $219/month, then $232.74 in October–December."
-us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Computed gross income as $869.83/month from self-employment and disability benefits only, omitting the head's $602.58/month SSI. It also used a $200 standard deduction instead of $209. Its net income of $496.50 compares with the correct $1,090.08, which yields $219/month rather than $397.05."
-us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit-receipt input and set it to 0 instead of computing it, even though the prompt tells models to assume program take-up. Computing it gives net income of $1,090.08 and a benefit of $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,100/month at the couple rate instead of the head's individual $602.58/month. It also used a ~$230 standard deduction instead of $209 and a $536 maximum instead of $546. Its net income of ~$1,566 compares with the correct $1,090.08, which gives $219/month rather than ~$66."
-us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed counted income was too high for SNAP without computing it. Gross income of $1,472.42 is 84% of the guideline and net income of $1,090.08 is 62%, so both tests pass. The benefit is $546 − $327 = $219/month, then $232.74 from October."
-us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit amount that defaults to 0 instead of computing the entitlement, even though take-up is assumed. The computed benefit from net income of $1,090.08 is $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Used annual gross income of $10,438 from self-employment and disability benefits only, omitting the head's $7,230.96/year ($602.58/month) of SSI. It also used a $208/month standard deduction instead of $209. Its net income of $488.50/month compares with the correct $1,090.08, which gives $219/month rather than ~$400."
-us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Its $340/month benefit implies net income of about $687. It excluded the head's $602.58/month SSI and claimed an 'allowable medical deduction', even though the $40/year of OTC expenses falls below the $35/month threshold. Correct net income is $1,090.08, giving $219/month and then $232.74."
-us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Its $70/month benefit reflects about $1,586 of net income, which comes from counting couple-rate SSI of about $1,099/month. The household's SSI is the head's individual $602.58/month, which gives net income of $1,090.08 and $546 − $327 = $219/month (then $232.74)."
-us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate (~$1,099/month) instead of the head's individual $602.58/month, which produced a $70 monthly allotment. The correct net income of $1,090.08 yields $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Estimated SSI of about $13,183/year at the couple rate instead of the head's individual $7,230.96/year ($602.58/month). That produced annual SNAP income of $23,621 and net income of $19,033 instead of $1,090.08/month, which cut the benefit to $842.10 instead of $219/month plus $232.74/month for October–December."
-us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Computed net income of $487.67/month from self-employment earnings and disability benefits alone, never adding the head's $602.58/month SSI. With SSI, net income is $1,090.08 and the $546 maximum less a $327 contribution gives $219/month, not ~$400."
-us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Its $842.10 result reflects monthly net income of about $1,586, which comes from counting SSI at the couple rate of ~$1,099/month. The household's SSI is the head's individual $602.58/month, which gives net income of $1,090.08 and $219/month (then $232.74)."
-us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Estimated SSI at the couple rate, giving countable income of $1,586.08. The household's SSI is only the head's $602.58/month individual payment ($994 FBR minus $391.42 countable), so net income is $1,090.08 and the benefit is $546 − $327 = $219/month, then $232.74 in October–December."
-us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed income and assets exceed SNAP limits. Gross income is 84% of the guideline, net income is 62%, and assets are $912. All tests pass, and the benefit is $546 − $327 = $219/month, then $232.74."
-us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Put monthly gross income at about $1,928 by including roughly $1,058/month of couple-rate SSI instead of the head's individual $602.58. It also used a $204 standard deduction instead of $209. Its net income of ~$1,551 compares with the correct $1,090.08, which gives $219/month rather than ~$71."
-us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Counted about $1,085/month of SSI at the couple rate, putting gross income at $1,955 instead of $1,472.42. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546. The correct net income of $1,090.08 yields $219/month, not $63."
-us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Counted $13,123/year of couple-rate SSI instead of the head's individual $7,230.96/year ($602.58/month), which drove net income to $18,973/year. The correct monthly net income of $1,090.08 gives $219/month for January–September and $232.74 for October–December."
-us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Omitted the head's $602.58/month SSI and mixed periods by subtracting a single month's $209 standard deduction from annual income. It then took 30% of that $8,149 'annual net' against a $6,600 annual maximum. The correct monthly figures are gross income of $1,472.42 and net income of $1,090.08, giving $219/month."
-us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Put gross income at about $870/month from self-employment and disability benefits and left out the head's $602.58/month SSI. Its net income of about $488 compares with the correct $1,090.08, which gives $219/month rather than $399."
-us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so nothing was scorable. The correct derivation counts $866.67 of earnings, $3.17 of benefits, and $602.58 of SSI, giving net income of $1,090.08. That yields $219/month for January–September and $232.74 for October–December, totaling $2,669.22."
-us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Estimated SSI at $13,183/year, the couple rate with both spouses as recipients, instead of the head's individual $7,230.96/year ($602.58/month). That produced monthly net income of $1,586.08 instead of $1,090.08 and a benefit of ~$70/month instead of $219, plus $232.74/month for October–December."
-us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Claimed the $912 of bank assets exceeds the ~$4,500 disabled-household asset limit, but $912 is well below that limit, so the asset test passes. With net income of $1,090.08, the benefit is $546 − $327 = $219/month, then $232.74."
-us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted only $867 of earnings and $3 of unearned income per month and omitted the head's $602.58/month SSI. Its net income of about $488 compares with the correct $1,090.08, which gives $219/month rather than ~$400."
-us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Excluded all income of members it deemed SSI-eligible, but SNAP has no such exclusion. The head's $866.67/month of earnings and $602.58/month of SSI both count, giving net income of $1,090.08. It also used a $635 maximum instead of the FY2026 two-person maximum of $546."
-us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted ineligibility without applying any test. The household passes the gross (84% of FPL), net (62% of FPL), and asset tests and is categorically eligible through SSI. The benefit is $546 − $327 = $219/month, then $232.74 from October."
+us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Never computed or counted the head's $602.58/month SSI as SNAP unearned income, so it used about $870/month of gross income instead of $1,472.42. It also guessed a $540 maximum allotment instead of $546, landing at $380/month instead of $219."
+us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate of about $1,099/month as if both spouses were SSI recipients. The engine's SSI is $602.58/month (the $994 individual rate for the head, minus $391.42 of countable earnings). That overstated net income at $1,586 instead of $1,090.08 and cut the benefit to $70/month instead of $219."
+us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Wrongly cut self-employment income to about $809 using a self-employment-tax factor, then called the household ineligible without any calculation. With $1,472.42/month of gross income (including $602.58 of SSI), net income is $1,090.08, below the $1,762.50 net-income limit, so the household is eligible for $219/month."
+us,scenario_057,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Asserted a zero benefit without computing net income. The $912 in assets passes the asset test, and net income of $1,090.08 is 62% of the poverty guideline, so the household gets $546 − $327 = $219/month."
+us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of SNAP income. It also subtracted a monthly $219 standard deduction from annual income. Its own arithmetic produced $3,990, but it then submitted $6,240 with no justification."
+us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Left SSI out of SNAP income and treated the $38/year of disability benefits as $456/year of unearned income. It then raised its own $4,536 estimate to $5,736 for medical deductions, which don't apply because the $40/year of OTC expenses is far below the $35/month threshold."
+us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Claimed net income was near zero, ignoring the $866.67/month of self-employment income and the $602.58/month of SSI. It also used an invented $304/month maximum allotment instead of the $546 FY2026 two-person maximum."
+us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of $1,098.58/month at the couple rate. The engine pays only the head $602.58/month at the $994 individual rate, so net income is $1,090.08, not $1,586, and the benefit is $219/month, not $70."
+us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Left the $602.58/month of SSI out of SNAP income. It also took an unlisted 40% business-expense deduction off the $10,400 of self-employment income, which SNAP counts in full here, and used a $535 maximum allotment instead of $546."
+us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Reduced self-employment income with a self-employment-tax deduction that SNAP doesn't use, left out the $602.58/month of SSI, and treated the $38/year of disability benefits as $38/month. Its stated net income of about $2,000/month doesn't match its $297/month benefit."
+us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Added SSI of about $1,099/month at the couple rate. The engine pays only the head $602.58/month at the individual rate, so net income is $1,090.08, not $1,586, and the benefit is $219/month, not $70."
+us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Used only earnings and disability benefits (about $870/month) as gross income and never added the head's $602.58/month of SSI. That understated net income at $488.51 instead of $1,090.08. It also used $536 and $208 in place of the $546 maximum and the $209 standard deduction."
+us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Left the $602.58/month of SSI out of SNAP gross income, so its net income was $489.34 instead of $1,090.08. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546."
+us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted SSI of $1,067.17/month at the couple rate instead of the head's $602.58 individual-rate SSI, and treated the $38/year of disability benefits as $38/month. That pushed net income to $1,594.50 and the benefit down to $57.65/month."
+us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Never added the head's $602.58/month of SSI to SNAP income, so net income came out at $492.50 instead of $1,090.08 and the benefit at $398/month instead of $219."
+us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted $13,122.96/year of SSI at the couple rate instead of the head's $602.58/month individual-rate SSI. It also took a $60 excess medical deduction that doesn't exist (OTC expenses of $40/year are below the $35/month threshold) and used a $6,600 annual maximum instead of $546/month."
+us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Declared the household ineligible without any computation. Net income of $1,090.08 is well under the $1,762.50 net-income limit, which yields $219/month for most of the year."
+us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Included SSI at the roughly $1,099/month couple rate instead of the head's $602.58/month individual-rate SSI. That overstated net income and gave $76/month instead of $219."
+us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729/year of unearned income, which is couple-rate SSI for both spouses. The engine's SSI is only the head's $602.58/month ($7,230.96/year), so net income is $1,090.08/month and the benefit is $219/month."
+us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income and assets block SNAP with no calculation. The $912 in assets passes the asset test, and net income of $1,090.08 is 62% of the poverty guideline, which yields $219/month."
+us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave $3,216 ($268/month) with no derivation. The correct calculation counts the $602.58/month of SSI alongside $866.67 of earnings, giving net income of $1,090.08 and $219/month (rising to $232.74 for October–December), for $2,669.22."
+us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave $3,384 ($282/month) with no derivation. This understates countable income: including the $602.58/month of SSI gives net income of $1,090.08 and a $219/month allotment for most of the year, totaling $2,669.22."
+us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Built gross income from only the $10,400 of self-employment income and $38 of disability benefits and never counted the head's $602.58/month of SSI. It also used a $200 standard deduction instead of $209, giving net income of $496.50 instead of $1,090.08."
+us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit-receipt input set to 0 instead of a benefit to compute. With take-up assumed, the household's net income of $1,090.08 yields $219/month."
+us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,100/month at the couple rate instead of the head's $602.58/month individual-rate SSI, and used a $230 standard deduction instead of $209. That inflated net income to $1,566 and cut the benefit to $66/month."
+us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed counted income was too high without computing it. Gross income of $1,472.42 is 84% of the poverty guideline and net income of $1,090.08 passes the 100% net-income test, which yields $219/month."
+us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SNAP as a receipt amount that had to be given in the facts instead of computing it under the take-up assumption. The household's net income of $1,090.08 produces a $219/month allotment."
+us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month of SSI out of SNAP income, using $10,438/year of gross income instead of $17,669. That understated net income and produced about $400/month instead of $219."
+us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,Applied a medical deduction that doesn't exist ($40/year of OTC expenses is below the $35/month threshold) and left the $602.58/month of SSI out of countable income. The result was $340/month instead of $219.
+us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"A $70/month benefit implies net income of about $1,586, which counts couple-rate SSI of about $1,099/month. The engine pays only the head $602.58/month at the individual rate, giving net income of $1,090.08 and $219/month."
+us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate (about $1,099/month), giving a $70/month allotment. The head's individual-rate SSI of $602.58 gives net income of $1,090.08 and $219/month."
+us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Its $23,621 of SNAP income includes about $13,183/year of couple-rate SSI. The engine's SSI is the head's $602.58/month ($7,230.96/year), so annual net income is about $13,081, not $19,033."
+us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Computed net income of $487.67/month from earnings and disability benefits alone and left out the head's $602.58/month of SSI, which brings net income to $1,090.08 and the benefit down to $219/month."
+us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Its $842.10 answer reflects couple-rate SSI of about $1,099/month in net income ($1,586). The head's individual-rate SSI is $602.58/month, so net income is $1,090.08 and the benefit is $219/month."
+us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Its net income of $1,586.08 includes couple-rate SSI of about $1,099/month instead of the head's $602.58/month individual-rate SSI. That overstated net income by about $496 and cut the allotment to $70/month."
+us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that income and assets exceed SNAP limits without computing them. Assets of $912 and net income of $1,090.08 both pass, giving $219/month."
+us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Its monthly gross of about $1,928 includes roughly $1,058 of couple-rate SSI instead of the head's $602.58 individual-rate SSI, so net income came out at $1,551 instead of $1,090.08. It also used a $204 standard deduction instead of $209."
+us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Its monthly gross of about $1,955 includes about $1,085 of couple-rate SSI. The head's individual SSI is $602.58, so gross income is $1,472.42 and net income is $1,090.08. It also used a $536 maximum and a $204 standard deduction instead of $546 and $209."
+us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Counted $13,123/year of couple-rate SSI instead of the head's $7,230.96/year ($602.58/month) of individual SSI, which inflated net income to $18,973/year and shrank the benefit to $860."
+us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Subtracted the $209 monthly standard deduction from annual income in the same calculation as an annual $6,600 maximum, and left out the $602.58/month of SSI. The correct monthly calculation gives net income of $1,090.08 and $219/month."
+us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Used about $870/month of gross income and never added the head's $602.58/month of SSI. That gave net income of $488 instead of $1,090.08 and a $399/month benefit instead of $219."
+us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score."
+us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Counted $13,183/year of couple-rate SSI. The engine's SSI is the head's $602.58/month at the $994 individual rate, so net income is $1,090.08, not $1,586.08, and the benefit is $219/month, not about $70."
+us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Claimed $912 of bank assets exceeds a $4,500 limit, which is backwards: $912 is below the limit, and the household also has categorical eligibility through SSI. With the asset test passed, net income of $1,090.08 yields $219/month."
+us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted only $867 of earnings and $3 of disability benefits as income and omitted the head's $602.58/month of SSI. That gave net income of $488 instead of $1,090.08 and about $400/month instead of $219."
+us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,Wrongly excluded all income of SSI-eligible members from SNAP. SNAP counts their self-employment income and the $602.58/month of SSI. It also used a $635 maximum allotment instead of $546.
+us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded the household gets no SNAP without any income test. Gross income of $1,472.42 (84% of the poverty guideline) and net income of $1,090.08 (62%) both pass, giving $219/month."
us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold."
us,scenario_057,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that income exceeds Louisiana's Medicaid limits, but household MAGI is 0.45 x FPL — roughly $10,438 against a 138% FPL expansion cutoff near $29,200 for a household of two in 2026 — so it compared the $10,400 self-employment figure against a threshold far lower than the one the ACA adult expansion actually applies. It also gave the spouse's disability status weight while never running the only test that governs a 20-year-old in an expansion state: MAGI at or below 138% FPL under the ADULT category."
us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid as requiring an affirmative qualifying attribute in the fact pattern and defaulted to 0 in its absence, missing that Louisiana's ACA adult expansion makes age 19–64 plus MAGI at or below 138% FPL sufficient on its own. The listed facts fully determine eligibility — spouse age 20, non-dependent, household MAGI at 0.45 x FPL — so the ADULT expansion category resolves to eligible without any additional detail."
@@ -4973,101 +5065,104 @@ us,scenario_057,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,Fa
us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model said being disabled satisfies the under-65 Medicare pathway. That pathway requires 24 months of SSDI entitlement (or ESRD/ALS), not disabled status alone. The spouse has no SSDI months, so under-65 eligibility fails."
us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility."
us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present."
-us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted a placeholder 0 without doing any SSI calculation. The head is an eligible blind individual with $912 in resources, under the $2,000 limit, and has $391.42/month of countable income after the $20 general exclusion, the $65 earned-income exclusion, and the one-half earnings disregard. That gives $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated the spouse as SSI-disabled and applied the eligible-couple FBR of $17,892/year to pooled couple income of $4,709. The spouse is not aged/blind/disabled for SSI, so the head is paid from the $11,928 individual FBR. The spouse's $24/year is not deemed, and the head's own countable income is $4,697, which gives $7,231."
-us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Left out the one-half earned-income disregard, counting $9,620 of earnings instead of about half that, and treated the $14 and $24 annual disability payments as monthly amounts. It then called the household ineligible even though its own $10,076 was below the couple rate it cited. The correct count is ($866.67 − $18.83 − $65) / 2 = $391.42/month against the $994 individual FBR, giving $7,231."
-us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Counted the 20-year-old spouse as a qualifying disabled individual and subtracted pooled income from an assumed couple FBR of about $17,604. The spouse fails the SSI aged/blind/disabled test, so only the head is eligible, at the $994/month individual rate. With $391.42/month of countable income, the head receives $7,231/year."
-us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Applied an assumed eligible-couple FBR of $1,538/month because it treated both spouses as SSI-disabled, then replaced its own $14,271.50 result with an unexplained $13,628. The spouse is not SSI aged/blind/disabled, so the head is paid $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Said countable self-employment income of about $9,600 exceeded the benefit rate, which means it never applied the one-half earned-income disregard. After the exclusions, the head's countable income is only $391.42/month, well below the $994 individual FBR, so SSI is $602.58/month, or $7,231/year."
-us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the household as an eligible couple, used the $1,491 couple FBR, and pooled the spouse's $2/month into unearned income, giving $392.42 of countable income. The spouse is not SSI aged/blind/disabled and that $2/month is below the $497 deeming threshold. The head's countable income is therefore $391.42 against the $994 individual FBR, or $7,231/year."
-us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Used an eligible-couple FBR of $1,486/month because it assumed the spouse was SSI-eligible. Its final pass also dropped the unused general exclusion and counted $18 of unearned income plus $4,810 of earned income. The head alone is eligible at the $994 individual rate with $4,697/year of countable income, giving $7,231."
-us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Its own reasoning reached about $12,590 using a $1,450 couple FBR, but it submitted 0, which contradicts that work. It also treated the spouse as SSI-eligible. Only the head qualifies, at the $994 individual FBR less $391.42 of countable income, which is $7,231/year."
-us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491/month eligible-couple rate because it treated the spouse as SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head is an eligible individual with an ineligible spouse. The head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Paid both spouses separate individual benefits, including about $991/month for the spouse, and added them together. The spouse is not SSI aged/blind/disabled and receives $0. The head's countable income is $391.42/month, not about $430, so the head alone receives $602.58/month, or $7,231/year."
-us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Subtracted $392.42 of pooled countable income from an assumed couple FBR of $1,483 because it treated the spouse as SSI-eligible. The spouse is not aged/blind/disabled for SSI, so the head's own $391.42 is subtracted from the $994 individual FBR, giving $7,231/year."
-us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Used a $1,486 couple FBR by treating both spouses as SSI-eligible. It also read the annual $38 of disability benefits as monthly and counted $18/month of unearned income, reaching $418.83. The head alone is eligible at $994/month with $391.42 of countable income, which is $7,231/year."
-us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR and pooled the spouse's unearned income into the couple's countable income. The spouse fails the SSI aged/blind/disabled test, so the head receives $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Treated the household as a disabled couple and used a $1,486/month couple maximum with $4,709 of pooled countable income. The spouse is not SSI aged/blind/disabled, so the head's own $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231."
-us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Said the $10,400 of self-employment earnings made the household ineligible, without applying the $65 earned-income exclusion and the one-half disregard. Those leave only $391.42/month of countable income against the $994 individual FBR, so SSI is $7,231/year."
-us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Used the couple maximum benefit because it assumed both spouses qualified. The spouse is not SSI aged/blind/disabled, so the head is paid from the $994/month individual FBR less $391.42 of countable income, or $7,231/year."
-us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Subtracted $4,709 of pooled countable income from a $17,400 couple FBR because it treated the spouse as SSI-eligible. Only the head qualifies, so the $11,928 individual FBR less the head's $4,697 of countable income gives $7,231."
-us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Said income and resources exceeded the SSI limits. The $912 in resources is below the $2,000 individual limit, and after exclusions the head's countable income of $391.42/month is below the $994 FBR, so SSI is $7,231/year."
-us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Calculated an eligible-couple benefit because it treated both adults as SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head receives the individual rate of $994/month less $391.42 of countable income, which is $7,231/year."
-us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the household as an eligible couple, which implies a $1,486/month couple FBR with pooled income. Only the head is SSI aged/blind/disabled, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Said SSI was not simulated and returned 0 instead of calculating it. The head is an eligible blind individual with $391.42/month of countable income against the $994 individual FBR, so SSI is $7,231/year."
-us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Assumed $10,400 of self-employment income and $912 of assets exceeded the SSI limits, and relied on the absence of listed SSI receipt even though take-up is assumed. The assets are under the $2,000 limit, and the exclusions bring countable income down to $391.42/month, which leaves a $7,231/year benefit."
-us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Used a couple FBR of $1,490.56 because it treated the spouse as SSI-disabled. It also applied the full $85 of exclusions to earnings, even though part of the general exclusion is first used by unearned income. The head alone is eligible at $994/month with $391.42 of countable income, or $7,231/year."
-us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Said the $912 in assets was above the SSI limit, but it is well under the $2,000 individual limit. It also said self-employment income ruled out eligibility, when the exclusions reduce countable income to $391.42/month and leave a $602.58/month benefit, or $7,231/year."
-us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned 0 because the prompt listed no SSI receipt, ignoring the instruction to assume program take-up. The head qualifies as an eligible blind individual, and the benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied the $17,892 eligible-couple rate and pooled both spouses' disability income on the assumption that both were SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231."
-us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied a couple federal benefit rate of about $1,450/month because it treated both adults as SSI-eligible. Only the head is SSI aged/blind/disabled, so the benefit is $994/month less $391.42 of countable income, or $7,231/year."
-us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Reduced the 2026 couple maximum on the assumption that both spouses met the SSI disability criteria. The spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse and receives $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Calculated an eligible-couple benefit of $13,183 because it treated the spouse as SSI-disabled. The spouse fails the aged/blind/disabled test, so only the head is paid, from the $994/month individual FBR, which gives $7,231/year."
-us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Used the $17,892 couple maximum and counted the spouse's $24 against the general exclusion. The spouse is not SSI aged/blind/disabled and that income is below the deeming threshold, so the head's $4,697 of countable income is subtracted from $11,928, giving $7,231."
-us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Compared $392.42 of pooled countable income with the $1,491 couple FBR because it treated both spouses as SSI-disabled. Only the head qualifies, so the $994 individual FBR less $391.42 gives $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Started from the $1,491/month couple benefit, which assumes an eligible couple. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
-us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Subtracted $4,709 of pooled countable income from the $17,892 couple maximum because it treated both spouses as SSI-disabled. The spouse is not SSI-eligible, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231."
-us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Said countable self-employment income exceeded the couple FBR, so it did not apply the $65 exclusion and the one-half earned-income disregard. The head's countable income is $391.42/month, below the $994 individual FBR, which leaves $7,231/year."
-us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Used a projected $1,486 couple FBR because it treated the spouse as SSI-disabled. Only the head is SSI aged/blind/disabled, so the $994 individual FBR less $391.42 of countable income gives $602.58/month, or $7,231/year."
-us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Applied the $17,832 couple FBR on the assumption that both spouses were eligible. It also left the unused $202 of the general exclusion off the earnings, counting $4,810 of income. The head alone is eligible, with $4,697 of countable income against the $11,928 individual FBR, which gives $7,231."
-us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Applied the $1,486/month couple rate because it treated both spouses as disabled SSI claimants. The spouse fails the SSI aged/blind/disabled test, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231."
-us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Used a couple FBR of $17,835, and it counted the $20 general exclusion twice by fully excluding the $38 of unearned income while also subtracting the full $1,020 from earnings. Only the head qualifies, so the $11,928 individual FBR less $4,697 of countable income gives $7,231."
-us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Used a $1,490/month couple rate because it treated the spouse as SSI-eligible, and it understated countable income at about $370/month. The head alone is eligible at $994/month with $391.42 of countable income, which gives $7,231/year."
-us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SSI value and no explanation. The correct derivation uses the $994/month individual FBR less the head's $391.42 of countable income, which is $602.58/month, or $7,231/year."
-us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $17,892 couple benefit rate because it treated both spouses as SSI-disabled. The spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse and receives $11,928 − $4,697 = $7,231."
-us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Concluded from the small disability benefit amounts that no SSI was payable, without running the SSI income test. The head is an eligible blind individual with $391.42/month of countable income against the $994 FBR, which gives $7,231/year."
-us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR because it treated both spouses as eligible, and it applied the full $85 of exclusions to earnings. Only the head qualifies, and the head's $1.17/month of unearned income uses part of the general exclusion, so countable income is $391.42 and SSI is $602.58/month, or $7,231/year."
-us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Moved between made-up couple FBRs of $979–$1,058 and individual FBRs of $674–$706, treated the annual disability payments as monthly, and submitted $8,472, which none of its calculations produced. The actual 2026 individual FBR is $994/month and the head's countable income is $391.42/month, which gives $7,231/year."
-us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded that income and resources produce no positive SSI benefit. The $912 in resources is below the $2,000 limit, and after the $20, $65, and one-half earned-income exclusions the head's countable income is $391.42/month, which leaves $602.58/month, or $7,231/year."
+us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"The model submitted a placeholder 0 and never computed SSI. The head alone qualifies at the $994/month individual rate. After the $20 general exclusion, the $65 earned-income exclusion and the one-half earnings exclusion, the head has $391.42/month of countable income, leaving a $602.58/month ($7,231/year) payment."
+us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated both spouses as an SSI-eligible couple and used the $1,491/month ($17,892/year) couple rate. Under PolicyEngine's rules the spouse is not SSI aged/blind/disabled, so the head gets the $994/month individual rate. The spouse's $2/month of income falls below the deeming threshold. The head's countable income is $391.42/month, giving $7,231/year."
+us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model left out the $20 general exclusion and the one-half exclusion of remaining earnings, counting $9,620 of self-employment income in full. It also claimed that $10,076 of countable income exceeds the $16,980 couple rate, which its own numbers contradict. The correct countable income is $391.42/month against the head's $994 individual rate, for $7,231/year."
+us,scenario_057,ssi,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model declared the household SSI-ineligible without any calculation. The head is blind and disabled with $912 in resources, under the $2,000 individual limit. After exclusions, the head's $866.67/month of earnings leaves only $391.42 countable against the $994 individual rate, so the head receives $7,231/year."
+us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model treated the spouse as SSI-eligible and used a couple rate of about $17,604/year. It also applied the $20 general exclusion twice, once against earnings and again against unearned income. The head is the only eligible individual and is paid $994/month minus $391.42 countable income, or $7,231/year."
+us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model used an inflated couple rate of $1,538/month for both spouses, and its final figure does not follow from its own subtraction. The spouse is not SSI aged/blind/disabled, so the head gets the $994/month individual rate minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"The model counted about $9,600 of self-employment income without the one-half exclusion of earnings left after the $20 and $65 exclusions. Correctly excluded, countable earnings are $391.42/month, well under the head's $994 individual rate, so the head receives $7,231/year rather than $0."
+us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model's exclusion arithmetic was nearly correct, but it subtracted countable income from the $1,491 couple rate because it treated the spouse as eligible. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model used a $1,486/month couple rate for both spouses. Its final pass also applied the $20 general exclusion on an annual basis to unearned income and dropped the unused portion that should carry over to earnings. The head alone is eligible at $994/month with $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model's own calculation produced a positive $12,590 couple benefit, yet it submitted 0. The calculation itself was also wrong: it used the $1,450 couple rate and did not carry the unused general exclusion over to earnings. The head alone qualifies at $994/month minus $391.42 countable income, or $7,231/year."
+us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model applied the $1,491/month couple rate on the premise that both spouses are SSI-eligible. Under PolicyEngine's rules the spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse. The benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model paid the spouse a nearly full $991/month individual benefit. The spouse fails the SSI aged/blind/disabled test and gets $0. The model also overstated the head's countable income at about $430 when the correct figure is $391.42. The head alone receives $602.58/month, or $7,231/year."
+us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model subtracted $392.42 of countable income from an estimated $1,483 couple rate, treating the spouse as eligible. Only the head qualifies, so the individual rate of $994/month applies, less $391.42 of the head's own countable income, for $7,231/year."
+us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model used the $1,486 couple rate for both spouses. It also counted $18 of unearned income instead of letting the $20 general exclusion absorb the $3.17 and carry the rest over to earnings. The head alone is eligible at $994/month with $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model applied the $1,491/month couple rate as though both spouses were SSI-eligible. The spouse is not SSI aged/blind/disabled, so the head receives the $994 individual rate minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated both spouses as eligible and used a $1,486/month couple rate ($17,832/year). Only the head passes the SSI aged/blind/disabled test, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"The model claimed the head's earnings rule out SSI. Applying the $20 and $65 exclusions and then excluding half the remainder leaves only $391.42/month countable against the $994 individual rate, so the head receives $7,231/year."
+us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model applied the couple maximum benefit on the premise that both spouses qualify. The spouse is not SSI aged/blind/disabled, so the head receives the $994/month individual rate minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from a $17,400 couple rate, treating both spouses as eligible. Only the head qualifies, so the individual rate of $11,928/year ($994/month) applies, less $4,697 of countable income, giving $7,231."
+us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model claimed income and resources exceed the SSI limits. The $912 in the bank is under the $2,000 individual limit, and countable income after exclusions is $391.42/month, below the $994 individual rate, so the head receives $7,231/year."
+us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model computed an eligible-couple benefit because it treated the spouse as SSI-disabled. Only the head passes the SSI aged/blind/disabled test, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated both adults as an eligible couple and paid a couple-rate benefit. The spouse is not SSI aged/blind/disabled, so the head alone receives $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model claimed SSI is not simulated for this household. The blind, disabled head meets the resource test and has $391.42/month of countable income against the $994 individual rate, for $7,231/year."
+us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model assumed the head's $10,400 of earnings exceeds the SSI limits and treated the absence of listed SSI receipt as zero benefit, although the prompt says to assume take-up. After exclusions, countable income is $391.42/month against the $994 individual rate, for $7,231/year."
+us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"The model used a couple rate of about $1,490.56 because it treated the spouse as SSI-eligible. Only the head qualifies, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model said the household's assets exceed the SSI limit, but $912 is under the $2,000 individual limit. It also treated $10,400 of earnings as disqualifying even though only $391.42/month is countable after exclusions, against the $994 individual rate. The head receives $7,231/year."
+us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model set SSI to 0 because no SSI receipt was listed, ignoring the instruction to assume take-up. The blind, disabled head qualifies at $994/month minus $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple rate, treating both spouses as eligible. The spouse fails the SSI aged/blind/disabled test, so the head is paid the $11,928/year individual rate less $4,697 of the head's own countable income, giving $7,231."
+us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model applied a couple rate (the 2025 level of about $1,450) for both spouses. Only the head is SSI-eligible, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model reduced the 2026 couple maximum on the premise that both spouses are SSI-disabled. The spouse is not SSI aged/blind/disabled, so the head alone receives $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model computed an eligible-couple benefit of $13,183. Only the head passes the SSI aged/blind/disabled test, so the individual rate of $994/month applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple maximum, treating the spouse as eligible. The head is an individual with an ineligible spouse, so the benefit is the $11,928 individual rate minus $4,697, or $7,231."
+us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model used the $1,491 couple rate against $392.42 of countable income. The spouse is not SSI aged/blind/disabled, so the head receives $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model reduced the $1,491/month couple rate, treating both spouses as eligible. Only the head qualifies, so the $994 individual rate minus $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple maximum. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is the $11,928/year individual rate minus $4,697, or $7,231."
+us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed countable self-employment income exceeds the couple rate. That only holds if the one-half earned-income exclusion is skipped. With it, the head has $391.42/month of countable income against the $994 individual rate, for $7,231/year."
+us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"The model used a projected $1,486 couple rate because it treated both spouses as eligible. Only the head is SSI-eligible, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"The model applied the $17,832 couple rate and did not carry the unused general exclusion over to earnings, producing $4,810 of countable income. The head alone qualifies at the $11,928 individual rate with $4,697 of countable income, for $7,231/year."
+us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,832 couple rate, treating the spouse as eligible. The spouse is not SSI aged/blind/disabled, so the head receives the $11,928 individual rate minus $4,697, or $7,231."
+us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model used a $17,835 couple rate and applied the $85/month of exclusions to earnings only. Only the head qualifies, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"The model used a couple rate of about $1,490 and understated countable earnings at about $370/month. The head alone is eligible at the $994 individual rate with $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SSI value and no reasoning. The correct derivation pays the head alone the $994/month individual rate minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple rate, treating both spouses as eligible. The spouse fails the SSI aged/blind/disabled test, so the head receives the $11,928 individual rate minus $4,697, or $7,231."
+us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"The model took the small disability-benefit amounts as evidence that the household gets no SSI, ignoring the instruction to assume take-up. The blind, disabled head qualifies at $994/month minus $391.42 of countable income, for $7,231/year."
+us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"The model used the $1,491 couple rate and applied the full $85 of exclusions to earnings without accounting for the head's $1.17 of unearned income. Only the head is eligible, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model cycled through invented rates between $674 and $1,058/month, treated the couple as jointly eligible, and submitted $8,472, which none of its own calculations produced. The correct individual rate is $994/month; less the head's $391.42 of countable income, that gives $7,231/year."
+us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model asserted that the household's income and resources produce no benefit. The head meets the $2,000 resource limit and, after the $20, $65 and one-half exclusions, has $391.42/month of countable income against the $994 individual rate, for $7,231/year."
us,scenario_057,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied Louisiana's correct post-reform 3% flat rate but subtracted only $9,000 as the ""MFJ standard deduction"" — that is the repealed $4,500-per-filer personal exemption amount, not the deduction. Under Act 11 of the 2024 Third Extraordinary Session, Louisiana's standard deduction for married filing jointly is $25,000 (indexed beginning 2026), which exceeds the household's ~$9,665 federal AGI, so Louisiana taxable income is zero. Its $665 base is arithmetic on the wrong subtraction, and 3% of a base that does not exist produced the $20."
us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It reconstructed the pre-2025 Louisiana regime on two counts: it subtracted $4,500-per-filer personal exemptions ($9,000) and applied a graduated first-bracket rate, both eliminated effective tax year 2025 when Louisiana moved to a 3% flat tax with a $25,000 MFJ standard deduction. Its cited ""1.75% on the first $12,500"" matches neither the current flat rate nor the former 1.85% first bracket, whose MFJ width was $25,000. The $25,000 standard deduction alone wipes out the ~$9,665 federal AGI, leaving $0 rather than a $665 taxable base."
us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the miss is a contract failure rather than a substantive rule error. The required derivation runs $10,400 of SSTB self-employment income to roughly $9,665 federal AGI after the deductible half of self-employment tax, then against Louisiana's $25,000 married-filing-jointly standard deduction under its 3% flat tax, giving $0."
-us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly noted that Louisiana has a 5% EITC match and that the 43-year-old head is in the eligible age range for the childless EITC. It then dropped that line of reasoning and submitted 0. The couple's earned income of about $9,665 after the half-SE-tax deduction puts the federal childless EITC at its $664 maximum, so the Louisiana match is $33.20."
-us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed Louisiana has no meaningful refundable individual credits and that the couple needs dependents to get one. It missed Louisiana's refundable EITC, which is 5% of the federal EITC and applies to childless filers. Here that is 5% × the $664 maximum childless federal EITC = $33.20."
-us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It wrongly treated the Louisiana EITC as requiring qualifying children and dismissed the self-employment income as too small. The childless federal EITC applies because the head is 43, and about $9,665 of net SE earned income reaches the $664 maximum. Louisiana's 5% match is therefore $33.20."
-us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It correctly identified the Louisiana EITC as 5% of the federal EITC. It then assumed a childless couple with $10,400 of SE income gets only a minimal federal EITC and rounded the result to zero. Earned income of about $9,665 after the half-SE-tax deduction is past the phase-in, so the federal credit is the full $664 and the Louisiana credit is $33.20."
-us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It acknowledged that Louisiana's EITC is 5% of the federal EITC but wrongly called it nonrefundable. The Louisiana EITC is refundable, so 5% × the $664 federal childless EITC = $33.20 counts as a state refundable credit."
-us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly stated that Louisiana has no state EITC. Louisiana pays a refundable EITC equal to 5% of the federal EITC. This couple qualifies for the $664 maximum childless federal EITC, so the Louisiana credit is $33.20."
-us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly claimed the Louisiana EITC is nonrefundable or requires dependents. The Louisiana EITC is a refundable 5% match of the federal EITC, and the federal credit here is the $664 childless maximum on about $9,665 of net SE earnings, so the Louisiana credit is $33.20."
-us,scenario_057,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It gave no reasoning beyond saying no Louisiana credit applies, which means it left out the refundable Louisiana EITC. The correct figure is 5% × the $664 federal childless EITC = $33.20."
-us,scenario_057,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match but used $632, the 2024 maximum childless federal EITC, instead of the 2026 maximum of $664. It then rounded $31.60 to $32; the correct figure is $33.20."
-us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match but used $649, the 2025 maximum childless federal EITC, instead of the 2026 maximum of $664. That produced $32.45 instead of $33.20."
-us,scenario_057,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It set the federal EITC to $0, which zeroed out the Louisiana match. The couple files jointly with a 43-year-old head, so they qualify for the childless EITC. About $9,665 of net SE earned income gives the $664 maximum, and 5% of that is $33.20."
-us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match to a federal EITC of $630. That is not the 2026 maximum childless credit of $664, which this couple receives at about $9,665 of earned income. It therefore submitted $31.50 instead of $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no reasoning and answered 0, which means it left out Louisiana's refundable EITC. That credit is 5% of the $664 federal childless EITC this couple earns, or $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It asserted without reasoning that no Louisiana refundable credit applies, so it missed the refundable Louisiana EITC. That credit is 5% × the $664 federal childless EITC = $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It answered 0 without analysis and left out the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC generated by the head's net self-employment earnings, or $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used the correct 5% Louisiana match but applied it to $649, the 2025 maximum childless federal EITC, instead of the 2026 maximum of $664. That gave $32.45 instead of $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct 5% Louisiana match but took the federal EITC to be $649, the 2025 childless maximum, instead of the 2026 maximum of $664. That gave $32.45 instead of $33.20."
-us,scenario_057,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"It ruled the couple ineligible because the spouse is 20. The childless EITC age test for a joint return is met when either spouse is 25 to 64, and the head is 43. The couple therefore gets the $664 federal EITC and a $33.20 Louisiana EITC."
-us,scenario_057,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Its federal EITC of $795.60 is the 7.65% phase-in rate applied to the full $10,400 of gross SE income. It never subtracted half of SE tax from earned income and never capped the credit at the $664 childless maximum. The correct Louisiana credit is 5% × $664 = $33.20."
-us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no Louisiana refundable credit is triggered, missing the refundable Louisiana EITC. That credit equals 5% of the $664 federal childless EITC the couple receives on about $9,665 of net SE earnings, or $33.20."
-us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It answered 0 without identifying any Louisiana credit, so it left out the refundable Louisiana EITC. The correct figure is 5% × the $664 federal childless EITC = $33.20."
-us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It treated the absence of children as disqualifying for any Louisiana refundable credit. The Louisiana EITC matches the federal childless EITC, which this couple receives at the $664 maximum, so the credit is 5% × $664 = $33.20."
-us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It found no Louisiana refundable credit eligibility and so missed the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC the couple earns, or $33.20."
-us,scenario_057,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,It computed the federal EITC correctly at $664 but applied a 3.5% Louisiana match rate instead of the current 5%. That produced $23.24 instead of $33.20.
-us,scenario_057,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,It answered 0 with no analysis and left out the refundable Louisiana EITC. That credit is 5% × the $664 federal childless EITC = $33.20.
-us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It claimed Louisiana has no applicable refundable income-tax credit. Louisiana's refundable EITC equals 5% of the federal EITC, and this couple's federal childless EITC is $664, so the credit is $33.20."
-us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It asserted that Louisiana offers no applicable refundable credit and so missed the refundable Louisiana EITC. That credit is 5% of the couple's $664 federal childless EITC, or $33.20."
-us,scenario_057,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It correctly set up the Louisiana EITC as 5% of the federal EITC but wrongly took the federal EITC to be $0. A joint filer with a 43-year-old head and about $9,665 of net SE earned income gets the $664 childless maximum, so the Louisiana credit is $33.20."
-us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so there is no parseable answer. The correct figure is 5% × the $664 federal childless EITC = $33.20."
-us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,It computed the federal EITC correctly at $664 but applied a 10% Louisiana match rate instead of the 5% rate. That doubled the credit to $66.40 instead of $33.20.
-us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It wrongly stated that Louisiana has no state individual income tax. Louisiana levies an income tax and pays a refundable EITC of 5% of the federal EITC, which here is 5% × $664 = $33.20."
-us,scenario_057,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It reasoned that zero state liability and no children rule out refundable credits. The Louisiana EITC is refundable regardless of liability and matches the federal childless EITC, so the credit is 5% × $664 = $33.20."
-us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly asserted that Louisiana has no state individual income tax. Louisiana has an income tax with a refundable EITC equal to 5% of the federal EITC, which is $33.20 on this couple's $664 federal childless credit."
-us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It answered 0 without analysis and left out the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC generated by the head's net self-employment earnings, or $33.20."
-us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It guessed about $16,400 of taxable Social Security instead of working the tier-2 formula. Because provisional income is only $152 over the $44,000 adjusted base, the formula gives $6,000 + 85% x $152 = $6,129.20. It also left out the OBBBA $6,000-per-senior deduction ($12,000 for the couple), which together with the ~$35,500 standard deduction wipes out AGI. Its own $540 figure was then arbitrarily inflated to $1,177 with no computation behind it."
-us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but used a TCJA-sunset standard deduction of $18,100. The 2026 MFJ standard deduction is $32,200, plus $1,650 per aged spouse, plus the $6,000-per-senior OBBBA deduction, for $47,500 in total. That brings taxable income to $0, not $13,629."
-us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but assumed TCJA expired: a $19,900 pre-TCJA standard deduction plus $10,600 of personal exemptions. OBBBA made TCJA permanent, so personal exemptions stay at zero, and 2026 deductions are $32,200 + $3,300 aged + $12,000 senior = $47,500. Taxable income is $0, not $1,229.20."
-us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but applied pre-TCJA sunset rules: a $16,000 standard deduction, $1,600 aged additions and $5,100 personal exemptions each, $29,400 in total. The permanent 2026 rules give $32,200 + $3,300 + the $12,000 OBBBA senior deduction, which leaves no taxable income."
-us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but assumed TCJA sunset: a $15,875 standard deduction, $10,125 of personal exemptions and $3,125 of senior additions, $29,125 in total. It missed that OBBBA kept the $32,200 MFJ standard deduction and added the $6,000-per-senior deduction. Total deductions of $47,500 zero out taxable income."
-us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It skipped the provisional-income worksheet and included $18,554 of Social Security, overstating AGI at $44,154 instead of $31,729.20. It also left out the $12,000 OBBBA senior deduction. Then it applied an $842 elderly credit, which is fully eliminated here because the couple's nontaxable Social Security exceeds the $7,500 initial amount. With correct AGI and deductions, taxable income is $0."
-us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but explicitly used a projected pre-TCJA standard deduction plus personal exemptions (about $29,600). OBBBA made the TCJA structure permanent: the $32,200 MFJ standard deduction plus $3,300 of aged additions plus the $12,000 senior deduction exceeds AGI, so tax is $0, not $213."
-us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but used TCJA-sunset figures: a $15,300 standard deduction, $3,100 of elderly additions and $10,000 of personal exemptions. The actual 2026 amounts are $32,200 + $3,300 + the $12,000 OBBBA senior deduction, with no personal exemptions, which reduces taxable income to $0."
-us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security ($6,129.20) and AGI ($31,729.20) correctly. It then subtracted a standard deduction plus personal exemptions totaling only $29,900, which reflects the expired-TCJA regime. OBBBA's permanent 2026 rules allow $32,200 + $3,300 aged + $12,000 senior = $47,500 with no exemptions, leaving taxable income of $0, not $1,829.20."
-us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It botched the Social Security worksheet, reporting $12,129 taxable instead of $6,000 + 85% x $152 = $6,129.20, so its AGI was $37,729 instead of $31,729.20. It also assumed pre-TCJA law ($19,750 standard deduction plus $10,600 of exemptions) instead of the permanent $32,200 + $3,300 + $12,000 senior deduction, which leaves no taxable income."
-us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It misstated gross Social Security as $37,504 and took a flat 85% share ($16,876) instead of running the provisional-income worksheet, which gives $6,129.20, so AGI came out at $42,476 instead of $31,729.20. It used an outdated $29,200 + $3,100 standard deduction and left out the $6,000-per-senior OBBBA deduction. The correct $47,500 of deductions exceeds AGI, so tax is $0."
-us,scenario_059,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It capped taxable Social Security at the 85% maximum ($31,538) instead of applying the tier formula ($6,129.20). It used only the $32,200 base MFJ deduction, without the $3,300 aged additions or the $12,000 senior deduction. It then invented a residential clean energy credit from a mortgage balance and added that credit to tax instead of subtracting it. With correct AGI of $31,729.20 against $47,500 of deductions, tax is $0."
+us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model knew about Louisiana's 5% EITC match and that the 43-year-old head meets the age test, but it treated the federal childless EITC as negligible and submitted 0. Earned income of about $9,665 is past the phase-in ceiling and below the joint phase-out start, so the federal EITC is the full $664 and the Louisiana credit is $33.20."
+us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Louisiana offers no applicable refundable credits and missed Louisiana's refundable EITC, which equals 5% of the federal EITC. The couple gets the full $664 federal childless EITC, so the Louisiana credit is $33.20."
+us,scenario_057,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model assumed a refundable credit needs qualifying children or wage income. Self-employment income counts as earned income for the federal childless EITC ($664 here), and Louisiana refunds 5% of it, which is $33.20."
+us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model said Louisiana's earned income credit needs children and called the $10,400 of self-employment income too small. The childless federal EITC reaches its $664 maximum once earned income passes about $8,680, and this household has about $9,665, so Louisiana's 5% match is $33.20."
+us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model recognized the 5% Louisiana match on the federal childless EITC but called the federal credit minimal and rounded the state credit to zero. Net earned income of about $9,665 gives the full $664 federal maximum, so the state credit is $33.20, not zero."
+us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model called Louisiana's 5% EITC nonrefundable or limited, but it is refundable and is paid even when there is no state tax liability. Applied to the $664 federal childless EITC, it equals $33.20."
+us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said Louisiana has no state EITC in 2026. Louisiana does have a refundable EITC at 5% of the federal credit, which gives $33.20 on the household's $664 federal EITC."
+us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model called Louisiana's EITC-like credit generally nonrefundable and said the household lacks qualifying dependents. Louisiana's EITC is refundable and also covers childless filers, at 5% of the $664 federal childless EITC, or $33.20."
+us,scenario_057,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model said no refundable Louisiana credit applies and missed Louisiana's refundable EITC at 5% of the federal EITC. With the full $664 federal childless credit, that is $33.20."
+us,scenario_057,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used an outdated $632 federal childless EITC maximum (the 2024 value) instead of the 2026 maximum of $664, then rounded 5% of it to $32. The correct figure is 5% × $664 = $33.20."
+us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used the 2025 federal childless EITC maximum of $649 instead of the 2026 maximum of $664. That gave $32.45 instead of 5% × $664 = $33.20.
+us,scenario_057,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model set the federal EITC to $0. On a joint return, the 43-year-old head meets the 25–64 age test, and about $9,665 of net self-employment earnings gives the full $664 childless credit. Louisiana's 5% refundable match is therefore $33.20."
+us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but used a $630 federal EITC, an outdated childless maximum, instead of the 2026 maximum of $664. That gave $31.50 instead of $33.20."
+us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credits apply and missed Louisiana's refundable EITC, which equals 5% of the $664 federal childless EITC, or $33.20."
+us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no Louisiana refundable credit. The couple qualifies for the $664 federal childless EITC, and Louisiana refunds 5% of it, or $33.20."
+us,scenario_057,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model reported no Louisiana refundable credits and missed the refundable state EITC at 5% of the household's $664 federal EITC, which is $33.20."
+us,scenario_057,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model used the 2025 federal childless EITC maximum of $649 instead of the 2026 maximum of $664. That gave $32.45 instead of $33.20.
+us,scenario_057,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,The model used the 2025 federal childless EITC maximum of $649 instead of the 2026 maximum of $664. That gave $32.45 instead of 5% × $664 = $33.20.
+us,scenario_057,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model ruled the couple out of the EITC because the spouse is under 25. For married filing jointly, the childless age test needs only one spouse to be 25–64, and the 43-year-old head meets it. That gives the $664 federal credit and the $33.20 Louisiana match."
+us,scenario_057,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model computed a $795.60 federal EITC by applying the 7.65% phase-in rate to the full $10,400. It did not cap the credit at the 2026 childless maximum of $664 and did not subtract half of SE tax from earned income. With the cap, 5% of $664 is $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no Louisiana refundable credits and missed the refundable state EITC at 5% of the household's $664 federal childless EITC, which is $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits apply and missed the 5% refundable state EITC on the $664 federal childless EITC, which is $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model assumed a household without children cannot qualify for a Louisiana refundable credit. Louisiana's refundable EITC matches the federal childless EITC at 5%, and here that is 5% × $664 = $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model found no Louisiana refundable credit eligibility and missed that self-employment earnings qualify the couple for the $664 federal childless EITC. Louisiana refunds 5% of that, or $33.20."
+us,scenario_057,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model had the correct $664 federal EITC but used a 3.5% Louisiana match, the rate before 2018, instead of the current 5%. That gave $23.24 instead of $33.20."
+us,scenario_057,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said no state refundable credits apply and missed Louisiana's refundable EITC at 5% of the $664 federal childless EITC, which is $33.20."
+us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said Louisiana offers no applicable refundable credits, but Louisiana's EITC is refundable at 5% of the federal EITC. That gives $33.20 on the household's $664 federal credit."
+us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits apply and missed the refundable state EITC at 5% of the $664 federal childless EITC, which is $33.20."
+us,scenario_057,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model set the federal EITC to $0. On a joint return, the 43-year-old head meets the childless age test, and about $9,665 of earned income gives the full $664 credit. Louisiana's 5% match is therefore $33.20."
+us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was no answer to grade against the $33.20 Louisiana EITC."
+us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model had the correct $664 federal EITC but used a 10% Louisiana match instead of the actual 5%. That gave $66.40 instead of $33.20.
+us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Louisiana has no individual income tax. Louisiana does levy a state income tax and offers a refundable EITC at 5% of the federal EITC, which is $33.20 here."
+us,scenario_057,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model assumed zero liability and no children rule out any refundable credit. Louisiana's EITC is refundable regardless of liability and covers childless filers, giving 5% × $664 = $33.20."
+us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model claimed Louisiana has no individual income tax. Louisiana taxes individual income and pays a refundable EITC at 5% of the federal EITC, which is $33.20 on the $664 federal credit."
+us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits apply and missed the refundable state EITC at 5% of the household's $664 federal childless EITC, which is $33.20."
+us,scenario_059,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model treated taxable Social Security as roughly 85% of benefits. The tiered formula gives only $6,129.20 because provisional income is just $152 over $44,000. It also failed to net the deductions correctly: AGI of $31,729.20 minus the $35,500 standard deduction (with age add-ons) and the $12,000 senior deduction is below zero. Its claimed taxable income of about $30,000 is impossible."
+us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model guessed taxable Social Security at about $16,400 instead of computing $6,000 + 85% × ($44,152 − $44,000) = $6,129.20. That inflated AGI to $42,000. It also left out the $12,000 OBBBA senior deduction, and then raised its own $540 result to $1,177 with no basis. The correct AGI of $31,729 is below even the $36,600 deduction it used."
+us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed AGI correctly at $31,729. It then used an $18,100 standard deduction instead of the 2026 MFJ amount, which is $32,200 plus $1,650 per spouse aged 65+ (TCJA levels made permanent by OBBBA), and it left out the $12,000 senior deduction. The full deductions exceed AGI, so taxable income is $0."
+us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA would sunset in 2026 and used a pre-TCJA standard deduction of $19,900 plus $10,600 in personal exemptions. OBBBA made the $32,200 MFJ standard deduction permanent, adding $3,300 in aged amounts and the $12,000 senior deduction. That total exceeds the $31,729.20 AGI and brings taxable income to $0."
+us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model applied sunset-era law: a $16,000 base standard deduction, $1,600 aged additions, and $5,100 personal exemptions, for $29,400 in total. Under OBBBA, the 2026 MFJ standard deduction is $32,200 plus $3,300 in aged additions plus the $12,000 senior deduction. That exceeds the $31,729.20 AGI, so tax is $0."
+us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model mixed a post-TCJA single-level standard deduction ($15,875) with reinstated personal exemptions, totaling $29,125. The correct 2026 MFJ standard deduction is $32,200 plus $1,650 for each spouse aged 65+, and each spouse also gets the $6,000 senior deduction. Those deductions wipe out the $31,729.20 AGI entirely."
+us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model counted $18,554 of Social Security as taxable instead of the tiered-formula amount of $6,129.20, which inflated AGI to $44,154. It also left out the $12,000 OBBBA senior deduction. It then invented an elderly credit, but that credit is unavailable because the couple's nontaxable Social Security exceeds the $7,500 initial amount. Correct deductions exceed the $31,729.20 AGI, so tax is $0."
+us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's AGI of $31,729 is correct. It then projected a pre-TCJA standard deduction plus personal exemptions of about $29,600, assuming the TCJA sunset. Under OBBBA, the 2026 MFJ standard deduction is $32,200, plus $3,300 for age and the $12,000 senior deduction, all of which exceed AGI."
+us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted a $15,300 standard deduction, $3,100 in elderly additions, and $10,000 of personal exemptions, a mix of sunset-era rules. Under the permanent TCJA rules in OBBBA, the MFJ standard deduction is $32,200 plus $3,300 for age, and there is also a $12,000 senior deduction. Together these exceed the $31,729 AGI, so tax is $0."
+us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model computed AGI correctly at $31,729.20. It then subtracted personal exemptions and a sunset-level standard deduction totaling $29,900. Personal exemptions stay at $0 under OBBBA, and the 2026 MFJ standard deduction of $32,200 plus $3,300 in aged additions plus the $12,000 senior deduction exceeds AGI."
+us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model doubled the base-tier taxable Social Security, reporting $12,129 instead of $6,000 + $129.20 = $6,129.20, which inflated AGI to $37,729. It also assumed the TCJA sunset and used a pre-TCJA standard deduction plus $10,600 in exemptions. The $35,500 OBBBA standard deduction with age add-ons plus the $12,000 senior deduction exceeds the true AGI."
+us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model applied a flat 85% to misadded gross benefits ($37,504 instead of $37,104), getting $16,876 of taxable Social Security instead of the tiered-formula amount of $6,129.20. It also used a $32,300 deduction built from outdated amounts, and it left out the $12,000 senior deduction. The correct AGI of $31,729.20 is fully offset."
+us,scenario_059,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the 85% cap ($31,538) as taxable Social Security instead of running the tiered formula, which gives $6,129.20. It also left out the $12,000 senior deduction. It then invented a residential clean energy credit tied to a mortgage balance, a credit that does not exist for this household, and added that credit to the tax instead of subtracting it."
us,scenario_059,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no payroll-taxable wages and therefore owed zero employee payroll tax, but then submitted $1,937 without any supporting computation. Its numeric output directly contradicts its own stated payroll-tax-base calculation."
us,scenario_060,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model correctly noted that the age-65 expansion applied only in 2021 but then applied the childless EITC phase-in rate anyway. At age 70, the head fails the under-65 age requirement for a taxpayer without qualifying children, so the $3,000 of wages generates no EITC rather than $229.50."
us,scenario_060,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly excluded the $45,600 of veterans benefits when asserting that the head passed Texas's aged Medicaid income test. The head does not satisfy the aged pathway or any other Medicaid category, so low bank assets and MAGI of 1.08 times FPL do not establish eligibility."
@@ -5139,36 +5234,37 @@ us,scenario_064,child1_wic_eligible,glm-5.3,parse_contract_failure,missing_outpu
us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that a 12-year-old is ""in the WIC-eligible age range,"" but WIC's child category ends at the fifth birthday, so a 12-year-old has no WIC categorical status at all — the model appears to have imported a school-age or free-school-meals age frame into WIC. It also skipped the WIC income test entirely, which caps countable income at 185% of the federal poverty guideline and disqualifies this household's ~$98,810 of income for 5 people regardless of age."
us,scenario_064,dependent1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly credited the head's and spouse's tax-unit MAGI to dependent1 as a claimed dependent and compared that total with a 100% FPL BadgerCare limit. The engine measures dependent1's own MAGI income level at 0.00 x FPL, which puts dependent1 in the ACA adult category as eligible."
-us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It denied eligibility by applying an adult/disability asset limit to the household's $56,000 bank account and $58,676 in stock. Dependent1 qualifies through the MAGI-based ACA adult category, which has no asset test, and their own income is 0.00 x FPL."
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1 against the head's ~$97k wages plus 401(k) distributions and partnership income. Dependent1's MAGI income level is 0.00 x FPL, well under the adult-category threshold, so they are eligible."
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It tested dependent1 against the household's income (the head's ~$97k wages and other income) under both MAGI and disability limits. The engine rates dependent1 in the ACA adult category with a MAGI income level of 0.00 x FPL, which is eligible."
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It held that the adult dependent's household income exceeds the Medicaid limits, which assigns the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which makes them eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It added the head's wages, partnership income, farm loss and 401(k) distributions into a household MAGI above 138% FPL and applied that total to dependent1. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 passes the adult expansion test."
-us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It judged that household income is above the adult limit, crediting the parents' income to dependent1. Dependent1's MAGI income level is 0.00 x FPL, which qualifies them for the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It compared household income with a 100% FPL BadgerCare adult limit, which charges the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, below any adult threshold, so they are eligible."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It declared that income exceeds Wisconsin's Medicaid levels even though dependent1 has $0 of income. The only income that exceeds the limit is the head's; dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no rule, only a bare 'not eligible'. That answer matches testing dependent1 against the high-income parents' household. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled that dependent1's household income exceeds the Wisconsin adult thresholds, which assigns the parents' income to dependent1. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 is eligible as an ACA expansion adult."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It denied eligibility 'due to household income', crediting the head's earnings to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which passes the ACA adult income test."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied family income to dependent1's adult Medicaid test. Dependent1's MAGI income level is 0.00 x FPL, so they are eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the tax unit's ~$100k+ income (the head's wages, partnership income and 401(k) distributions) as dependent1's MAGI household income and also cited dependent1's employer-sponsored insurance. The engine measures dependent1's MAGI income level at 0.00 x FPL and returns eligible in the ACA adult category, with the ESI coverage in place."
-us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for dependent1_medicaid_eligible. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible (1)."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled dependent1 ineligible because they live in a high-income household, which applies the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which makes them eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It defaulted to ineligible because there were no disability-income details and because of household income, treating disability as the only relevant pathway. Dependent1 qualifies through the MAGI-based ACA adult category, which needs no disability determination, at a MAGI income level of 0.00 x FPL."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly used the family's household MAGI for dependent1 as an adult tax dependent and found that total above Wisconsin's adult limit. The engine measures dependent1's MAGI income level at 0.00 x FPL, which is eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It applied income and resource rules as if dependent1 were being tested under a disability-based pathway. Dependent1 qualifies through the MAGI-based ACA adult category, which has no resource test, and their MAGI income level is 0.00 x FPL."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It cited the available income and assets as blocking eligibility, which means testing the household's bank and stock holdings and the parents' income. The MAGI-based ACA adult category has no asset test, and dependent1's MAGI income level is 0.00 x FPL, which is eligible."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed that income exceeds Medicaid thresholds even though dependent1 has $0 of income, so it counted the head's earnings. Dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It failed dependent1 on the ABD $2,000 resource test and on the MAGI adult income limit, charging dependent1 with the household's assets and income. The ABD pathway is irrelevant because dependent1 passes the MAGI-based ACA adult category, which has no asset test, at a MAGI income level of 0.00 x FPL."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly routed the 27-year-old to MAGI adult rules but counted household income against the adult limit and also applied ABD/SSI resource limits to household assets. Dependent1's own MAGI income level is 0.00 x FPL, and the adult category has no resource test, so they are eligible."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It placed dependent1 in the adult category but tested household MAGI, which includes the head's ~$97k wages and other income, against the adult limit. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 is eligible as an ACA expansion adult."
-us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It assigned dependent1 a household MAGI of ~273% FPL built from the parents' income and applied an ABD asset limit to the family's $100k+ of assets. Dependent1's own MAGI income level is 0.00 x FPL, and the ACA adult category has no asset test, so dependent1 is eligible."
-us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for dependent1_medicaid_eligible. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible (1)."
-us,scenario_064,dependent1_medicaid_eligible,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It answered 'Not eligible' with no rule. That result matches testing dependent1 against the high-income parents' household rather than dependent1's own $0 income. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible."
-us,scenario_064,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1 against family income of ~350% FPL and applied SSI-related tests to the head's $56k bank account and $58,676 in stock. Dependent1's MAGI income level is 0.00 x FPL, and the ACA adult category has no asset test, so dependent1 is eligible."
-us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It placed the head's $97,295 wages plus partnership and other income in dependent1's household MAGI and compared the total with a ~100% FPL adult limit. Dependent1's MAGI income level is 0.00 x FPL, below that limit or any other adult limit, so they are eligible."
-us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It stated that dependent1 has income too high to qualify, although dependent1's listed income is $0, so it imputed the parents' income to them. Dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category."
+us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the head's and spouse's tax-unit income toward dependent1's Medicaid eligibility and compared it to roughly $32k (100% FPL for 5). The engine sets dependent1's MAGI income level at 0.00× FPL, since this 27-year-old adult has no income, so they pass the Wisconsin adult limit easily and are eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility because the household's bank, stock and home assets exceed Medicaid asset limits. Dependent1 qualifies under the MAGI adult category, which has no asset test, and their MAGI income level is 0.00× FPL, so the assets are irrelevant and dependent1 is eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model judged dependent1 on household income against the disabled-group limit. The engine uses the MAGI adult category, where dependent1's own income level is 0.00× FPL, so dependent1 is eligible no matter how much the parents earn."
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model attributed the head's roughly $97k wages, 401(k) distributions and partnership income to dependent1's MAGI test. The engine evaluates this 27-year-old adult dependent at 0.00× FPL, which is below Wisconsin's adult threshold, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model measured dependent1 against total household income. The engine's MAGI adult test counts dependent1's own zero income (0.00× FPL), so dependent1 meets the Wisconsin adult income limit and is eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed household income counts toward this adult dependent. The engine computes dependent1's MAGI income level as 0.00× FPL under the adult category, which makes them Medicaid-eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model added up the head's wages, partnership, farm and 401(k) income as dependent1's household MAGI and found it above 138% FPL. The engine puts dependent1's MAGI at 0.00× FPL in the adult category, so they are below the threshold and eligible."
+us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income to this adult dependent. The engine's MAGI adult test uses dependent1's income level of 0.00× FPL, which qualifies them for Medicaid."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly identified Wisconsin's 100% FPL adult limit but compared it to household income. Dependent1's MAGI income level in the engine is 0.00× FPL, which is under that limit, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the household's income as exceeding dependent1's Medicaid limit. The engine evaluates dependent1 in the MAGI adult category at 0.00× FPL because they have no income, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reason for its 'not eligible' answer, which matches charging the parents' income to dependent1. The correct derivation places this 27-year-old with no income in the MAGI adult category at 0.00× FPL, which makes them eligible."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income to dependent1's adult Medicaid test. The engine computes dependent1's MAGI income level as 0.00× FPL, below the Wisconsin adult threshold, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility based on household income. The engine's adult-category test uses dependent1's MAGI income level of 0.00× FPL, which qualifies them."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model compared family income to the Wisconsin adult limit. The engine measures dependent1's MAGI at 0.00× FPL, so dependent1 falls under the adult limit and is eligible."
+us,scenario_064,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the head's $100k+ tax-unit income as dependent1's MAGI and treated the employer-sponsored insurance as disqualifying. The engine puts dependent1's MAGI income level at 0.00× FPL in the adult category, and having ESI does not bar Medicaid eligibility, so dependent1 is eligible."
+us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for dependent1_medicaid_eligible. The correct answer is eligible (1), because dependent1 is in the MAGI adult category at 0.00× FPL."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility because the household is high-income. The engine evaluates this adult dependent on their own MAGI income level of 0.00× FPL, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model defaulted to 'not eligible' because of disability-pathway details and high household income. The relevant pathway is the MAGI adult category, which needs no disability details. Dependent1's income level there is 0.00× FPL, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the family's household MAGI to dependent1 because they are a tax dependent. The engine sets dependent1's MAGI income level at 0.00× FPL for the adult category, which is below Wisconsin's limit, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model invoked income and resource rules to deny eligibility. The MAGI adult category has no resource test, and dependent1's MAGI income level is 0.00× FPL, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model relied on household income and assets. Dependent1 qualifies through the MAGI adult category, which has no asset test, at a MAGI income level of 0.00× FPL, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model said income exceeds the thresholds, using household income. Dependent1's own MAGI income level in the engine is 0.00× FPL, which is below the adult limit, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model said dependent1 fails both the ABD $2,000 resource test and the MAGI adult limit. The MAGI adult category has no resource test, and dependent1's MAGI income level is 0.00× FPL, so they pass the adult pathway and are eligible."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly chose the MAGI adult rules but counted household income against them, and added an irrelevant ABD resource test. The engine computes dependent1's adult-category MAGI at 0.00× FPL, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated household MAGI as dependent1's income for the adult test. The engine sets dependent1's MAGI income level at 0.00× FPL, so they are eligible under the adult category even without SSI."
+us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave dependent1 the household's roughly 273% FPL MAGI and also applied ABD asset limits. The engine evaluates dependent1 in the MAGI adult category at 0.00× FPL with no asset test, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for dependent1_medicaid_eligible. The correct answer is eligible (1), because dependent1 is in the MAGI adult category at 0.00× FPL."
+us,scenario_064,dependent1_medicaid_eligible,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its 'not eligible' answer matches charging the parents' income to dependent1. The correct derivation evaluates this 27-year-old with no income in the MAGI adult category at 0.00× FPL, which makes them eligible."
+us,scenario_064,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave dependent1 the family's roughly 350% FPL MAGI and also applied SSI-related asset tests. The engine measures dependent1's MAGI income level at 0.00× FPL in the adult category, which has no asset test, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly identified Wisconsin's roughly 100% FPL adult limit but counted the head's wages and partnership income as dependent1's MAGI. Dependent1's MAGI income level is 0.00× FPL, which is below that limit, so they are eligible."
+us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model said dependent1's income is too high, but dependent1 has no income. Their MAGI income level is 0.00× FPL, so they qualify under the adult category."
us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed 'SSDI-based eligibility' just because Dependent 1 is disabled, but the household lists no SSDI benefits, and unlisted amounts are 0. The under-65 disability path needs 24 months of actual SSDI entitlement, which Dependent 1 does not have, so the correct answer is not eligible (0)."
us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model said a disability alone qualifies someone for Medicare at any age. Under 65, Medicare instead requires 24 months of SSDI entitlement, or ESRD or ALS. Dependent 1 has no SSDI income, so a 27-year-old is not eligible."
us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the 24-month SSDI entitlement rule but then applied it without any SSDI receipt, using only the disability flag. The household has $0 in Social Security disability benefits and no months of receipt, so the SSDI path does not apply. A 27-year-old with no ESRD or ALS is not eligible."
@@ -5194,52 +5290,50 @@ us,scenario_064,dependent2_medicaid_eligible,grok-4.5,prompt_ambiguity,household
us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for dependent2_medicaid_eligible, so its answer is missing rather than wrong on substance. The correct answer is 0: household MAGI of 3.22× FPL exceeds Wisconsin's child limits, and no disability, pregnancy or caretaker pathway applies."
us,scenario_064,dependent2_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent2_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own chain reached about $4,430: AGI $114,422, less the $13,899 overtime and $510 QBI deductions, gives about $67,713 taxable and about $7,630 of tax, minus $2,200 CTC and $1,000 ODC. It then added an unsupported ~$1,529 of 'adjustments' and reported $5,959. No extra tax belongs in this measure; $7,639.29 − $3,200 = $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp income, even though it is part of that amount, and subtracted the $1,796 domestic production deduction, which was repealed after 2017. That raised AGI to $126,626 (vs $114,403.37) and QBI to $3,310 (vs 20% × $2,550 = $510), so taxable income was $77,217 instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $13,899 FLSA overtime premium to wages that already include it instead of taking the 2026 qualified overtime deduction. It also deducted the full $18,235 capital loss instead of the $3,000 limit and used the $14,600 single-filer standard deduction instead of the $32,200 MFJ amount. Its $18,285 tax on $98,486 is far above the ~$11,300 the 2026 MFJ brackets give, and it reached $9,631 through unexplained credits rather than the $3,200 CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction of $13,899. It also charged SE tax and deducted half of it ($989) on the $14,000 partnership income, although the $16,800 farm loss makes net SE earnings negative. It then treated only $1,072 of the CTC as nonrefundable, when the full $2,200 CTC and $1,000 ODC offset $7,639 of tax, and added an unexplained $1,000."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction and added $1,978 of self-employment tax into income tax. SE tax is excluded from this measure, and it is zero here because the $16,800 farm loss exceeds the $14,000 SE partnership income. It never subtracted the $3,200 CTC/ODC, so its $14,959 is higher than its own ~$11,178 total before credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction. It deducted a half-SE-tax amount (no SE tax exists because the farm loss makes net SE earnings negative) and the repealed $1,796 DPAD, and it inflated QBI to ~$3,900 (correct: $510). It counted only $2,500 of CTC/ODC instead of $3,200, then added an unexplained 'additional tax', so its $11,200 is higher than its own $8,700 tax before credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed AGI of $114,422 by deducting only the head's $108 traditional IRA contribution. The engine's IRA adjustment is $126.22 because it also deducts Dependent 1's $18 contribution, and it uses unrounded wages ($94,979.60) and overtime ($13,899.29). Those differences bring taxable income down to $67,794.08 and explain the whole $2.27 gap; the standard deduction, overtime and QBI deductions, and $3,200 of credits all match."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction of $13,899 and used a $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 and $24,800 figures. It also took a $1,560 SE health-insurance deduction and a $989 half-SE-tax deduction, neither of which applies because the head has employer coverage and the farm loss makes net SE earnings negative. It used a $2,000 CTC instead of $2,200, ending at $81,873 taxable vs $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction, deducted the repealed $1,796 DPAD and a half-SE-tax amount that does not exist here, and treated the CTC as mostly refundable. In fact the $2,200 CTC and $1,000 ODC fully offset tax against this liability. It then raised its ~$9,300 tax to $12,500 by adding Additional Medicare Tax and NIIT effects, which are outside this measure and do not apply at this income."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp income, which already contains it. That pushed AGI to ~$128.5k and QBI to ~$3.3k instead of $114,403 and $510. It also left out the $13,899 qualified overtime deduction, so taxable income was ~$92.9k instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income twice (business net $16,550 instead of $2,550), subtracted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200, and left out the $13,899 qualified overtime deduction. That gave $92,674 taxable vs $67,794.08. It also claimed the $2,000 CTC for the 18-year-old, who is past the CTC's under-17 age limit and gets only the $500 ODC, so its credits totaled $4,500 instead of $3,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied the pre-TCJA sunset rules: a $15,900 standard deduction, five $5,200 personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has no personal exemptions, a $32,200 MFJ standard deduction, a 12% bracket, a $2,200 CTC plus $500 ODC per other dependent, and the $13,899 overtime deduction. It also subtracted the $1,560 of health premiums from wages."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-TCJA sunset rules: five personal exemptions and a $1,000 CTC phasing out above $110,000. OBBBA extended TCJA permanently, giving a $32,200 MFJ standard deduction, no exemptions, 10%/12% brackets, a $2,200 CTC plus two $500 ODCs with a $400,000 phase-out, and a $13,899 overtime deduction. As a result it applied $150 of credits against $11,628.90 of tax instead of $3,200 against $7,639.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction, deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200, and applied a $2,000 CTC instead of the 2026 $2,200. Taxable income came to $81,366 rather than $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $19,648 of employer-sponsored insurance premiums from wages (taxable wages $75,332), although the stated wages are reduced only by the $2,315 traditional 401(k) deferral. It then itemized medical expenses and claimed $25,250 of personal exemptions and a $1,000 CTC under pre-TCJA rules. Under 2026 law there are no exemptions, the $32,200 standard deduction beats itemizing, the $13,899 overtime deduction applies, and credits are $2,200 CTC plus $1,000 ODC; its $4,717.20 is close to the correct figure only because the errors partly cancel."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation beyond mentioning itemized deductions. The correct path is AGI $114,403.37, less the $32,200 standard deduction (larger than any itemizable amount) and the $13,899.29 overtime and $510 QBI deductions, giving $67,794.08 taxable, $7,639.29 of tax, and $4,439.29 after $3,200 of CTC/ODC. Its $11,090.45 is exactly the bracket tax on about $96,554 of taxable income with no credits, which matches skipping the overtime deduction and the CTC/ODC and using small itemized deductions instead of the standard deduction."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset (~$26,000 of personal exemptions, a $16,200 standard deduction, a $1,000 CTC), subtracted ESI premiums and the repealed DPAD from income, and added a $2,000 10% early-distribution penalty. OBBBA keeps the $32,200 MFJ standard deduction with no exemptions and adds the $13,899 overtime deduction, and the credits are a $2,200 CTC plus $1,000 ODC. The §72(t) additional tax is a Schedule 2 'other tax' and is not part of income tax before refundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $94,756 AGI is $19,647 below the correct $114,403.37, which matches the $19,648 of employer-sponsored insurance premiums; those premiums do not reduce the stated wages. It then applied pre-TCJA personal exemptions and a $1,000 CTC. 2026 law has no exemptions and gives the $32,200 standard deduction, the $13,899 overtime and $510 QBI deductions, and $3,200 of CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions gives $67,794.08 taxable and $7,639.29 of tax; after $2,200 CTC and $1,000 ODC the result is $4,439.29. Its $11,940 is higher than the $9,307 pre-credit tax even with the overtime deduction dropped, so it left out both the overtime deduction and the $3,200 of credits and also overstated income."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. After the $3,200 of CTC/ODC it names, $6,215 implies ~$9,415 of tax before credits and ~$82,600 of taxable income, about $14,800 above the correct $67,794.08. That gap is the size of the $13,899 qualified overtime deduction for the FLSA overtime premium, which it left out."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. With the $3,200 of CTC/ODC it names, $4,324 implies $7,524 of tax before credits and ~$66,830 of taxable income, about $960 below the correct $67,794.08. It therefore took about $960 more in deductions than AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI allows."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. With $3,200 of CTC/ODC, $4,057 implies $7,257 of tax before credits and ~$64,610 of taxable income, about $3,190 below the correct $67,794.08. It therefore deducted about $3,190 beyond the only allowable items: the $32,200 standard, $13,899.29 overtime and $510 QBI deductions from AGI $114,403.37."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own chain reached about $67,713 taxable income (it used a $32,300 standard deduction instead of $32,200), about $7,630 of tax and $3,200 of CTC/ODC, which yields about $4,430. It then added an unexplained ~$1,529 of ""adjustments"" to get $5,959, so the final number does not follow from its own computation."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income, but the $14,000 is already a component of that total. This inflated net business income to $16,550 and the QBI deduction to $3,310; the correct figures are $2,550 and $510. It also deducted the $1,796 domestic production deduction, which was repealed after 2017, leaving $77,217 taxable income instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $13,899 overtime premium on top of wages that already include it and deducted the full $18,235 capital loss instead of the $3,000 limit. It used a $14,600 standard deduction instead of the 2026 MFJ $32,200, never took the OBBBA overtime deduction, and used a $2,000 CTC instead of $2,200 with no $1,000 ODC. It then applied an unexplained further reduction to reach $9,631."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no derivation and answered $0. The correct computation leaves $67,812.12 of taxable income after the $32,200 standard deduction, the $13,899 overtime deduction and the $510 QBI deduction, which produces $7,641.46 of tax. Only $3,200 of CTC/ODC offsets that, so the answer wrongly assumed deductions and credits wipe out all liability."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never took the 2026 OBBBA $13,899 qualified overtime deduction. It also computed SE tax on the $14,000 partnership income without netting the $16,800 farm loss (net SE earnings are negative, so there is no SE tax or half-SE deduction). It invented a split that made most of the $2,000 CTC refundable when the full $2,200 is nonrefundable here, then added $1,000 of unexplained ""adjustments""."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction. It also computed SE tax on $14,000 without netting the $16,800 farm loss and then added that SE tax into income tax, even though SE tax is not part of this measure. Finally it increased the total by unexplained amounts instead of subtracting the $3,200 CTC/ODC, ending at $14,959."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction and deducted the repealed $1,796 DPAD. It took a half-SE-tax deduction even though the farm loss makes net SE earnings negative, and used a ~$3,900 QBI deduction instead of $510. It then added an unspecified ""additional tax on other income"" and applied only $2,500 of credits instead of $3,200."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the 2026 OBBBA $13,899 overtime deduction and used the 2025 $30,000 standard deduction instead of $32,200. It also took a $1,560 SE health insurance deduction and a $989 half-SE-tax deduction even though netting the farm loss leaves no SE earnings, and used a $2,000 CTC instead of $2,200."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction, deducted the repealed $1,796 DPAD and took a half-SE-tax deduction even though net SE earnings are negative. It then treated the CTC as negligible and inflated the result to $12,500 for nonexistent Medicare/NIIT effects. The correct result subtracts the full $3,200 of CTC/ODC from $7,641.46 of tax."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp total, which pushed AGI to ~$128.5k instead of $114,421 and QBI to ~$3.3k instead of $510. It also omitted the $13,899 overtime deduction, so its taxable income (~$92.9k) far exceeds $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income (net business income $16,550 instead of $2,550), deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200 and omitted the $13,899 overtime deduction. It also claimed $4,000 CTC for two qualifying children, treating the 18-year-old as an under-17 child; the correct credits are $2,200 CTC plus two $500 ODCs."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law: a $15,900 standard deduction, $26,000 of personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent for 2026, so the correct rules are the $32,200 standard deduction, the 10%/12% brackets, a $2,200 CTC and $500 ODCs. It also subtracted health premiums from wages and omitted the $13,899 overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law, with personal exemptions and a CTC phaseout starting at $110,000, which left only $150 of credit. Under OBBBA, 2026 has no personal exemptions, a $32,200 standard deduction, a $13,899 overtime deduction and a $400,000 MFJ phaseout threshold, so the full $3,200 of CTC/ODC applies."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It reached the correct $114,422 AGI and $510 QBI deduction but omitted the 2026 $13,899 qualified overtime deduction. It also deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200 and used a $2,000 CTC instead of $2,200, leaving $81,366 of taxable income instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA rules: $25,250 of personal exemptions and a $1,000 CTC with no ODC. It also subtracted the $19,648 employer-sponsored premiums from the given wages and itemized medical expenses that included those employer premiums, and it treated the repealed $1,796 DPAD as a QBI deduction. It never applied the $32,200 standard deduction or the $13,899 overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no real derivation, and its $11,090 exceeds even the $7,641.46 pre-credit tax on the correct $67,812 taxable income. That means it overstated taxable income by leaving out the $13,899 overtime deduction and the correct $32,200 standard deduction, and it did not subtract the $3,200 CTC/ODC."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly used pre-TCJA 2026 rules: ~$26,000 of personal exemptions, a $16,200 standard deduction and a $1,000 CTC. OBBBA's permanent $32,200 standard deduction, $13,899 overtime deduction, $2,200 CTC and $500 ODCs apply instead. It also subtracted ESI premiums from wages, deducted the repealed DPAD and added a $2,000 early-withdrawal penalty that is not part of this regular-tax-less-nonrefundable-credits measure."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used personal exemptions and a $1,000 Child Tax Credit, which are pre-TCJA sunset rules, instead of 2026 OBBBA law. It also understated AGI at $94,756 instead of $114,421, omitted the $13,899 overtime deduction and the two $500 other-dependent credits, and did not use the $2,200 CTC."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation, and its $11,940 is above the $7,641.46 tax on the correct $67,812 taxable income even before credits. The answer therefore ignores the $13,899 overtime deduction and the $3,200 of CTC/ODC that reduce tax to $4,441.46."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $6,215 plus $3,200 of credits implies about $9,415 of pre-credit tax, or about $81,000 of taxable income. That matches the reference base without the 2026 $13,899 qualified overtime deduction, which it never mentions."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,324 plus $3,200 of credits implies about $7,524 of pre-credit tax, about $117 below the $7,641.46 due on $67,812.12. That means it understated taxable income by roughly $975 by taking deductions beyond the $32,200 standard, $13,899 overtime and $510 QBI deductions."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $4,057 plus $3,200 of credits implies about $7,257 of pre-credit tax, about $384 below $7,641.46, so taxable income came out roughly $3,200 too low. That gap matches overstated deductions: a QBI deduction on un-netted partnership income instead of 20% of the $2,550 net after the farm loss, or the repealed $1,796 DPAD."
us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and credits wipe out all tax. The only deductions are the $32,200 standard, $13,899.29 overtime and $510 QBI deductions; medical costs fall below the 7.5% AGI floor and never beat the standard deduction. That leaves $67,794.08 taxable and $7,639.29 of tax, of which the $3,200 CTC/ODC offsets only part."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It assumed deductions and credits eliminate all liability. After the $32,200 standard, $13,899.29 overtime and $510 QBI deductions, taxable income is still $67,794.08 with $7,639.29 of tax. The nonrefundable $2,200 CTC and two $500 ODCs remove only $3,200, leaving $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed AGI of $114,422 by deducting only the head's $108 IRA contribution. The engine's IRA adjustment is $126.22 because it also deducts Dependent 1's $18 contribution, and it carries unrounded wages ($94,979.60) and overtime ($13,899.29). That gives $67,794.08 taxable instead of $67,813 and explains the whole $2.27 gap; everything else matches."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"After its $3,200 of CTC/ODC, $2,219 implies $5,419 of tax before credits and ~$49,300 of taxable income, about $18,500 below the correct $67,794.08. It took ~$18,500 of deductions beyond the $32,200 standard, $13,899.29 overtime and $510 QBI deductions. The capital loss is capped at $3,000, and the $19,648 of ESI premiums does not reduce the stated wages."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,485 is $45.71 above $4,439.29, which equals ~$381 of extra taxable income at 12%. It used the right structure ($32,200 standard, overtime and QBI deductions, $3,200 of credits) but rounded its estimate instead of computing taxable income exactly as $114,403.37 − $32,200 − $13,899.29 − $510 = $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $2,200 CTC for Child 1 and left out the $500 credit for other dependents for Dependent 1 (27, disabled) and Dependent 2 (18, over the CTC age limit). Its tax before credits (~$7,655) was within ~$16 of $7,639.29, so the $1,015.71 overstatement comes almost entirely from the missing $1,000 ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored insurance premiums from wages and took the repealed $1,796 domestic production deduction, bringing AGI down to $92,978 instead of $114,403.37. The stated wages are reduced only by the $2,315 traditional 401(k) deferral, so taxable income is $67,794.08, not $46,369."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $5,570 equals $8,770 of tax on $77,217 of taxable income minus the $3,200 CTC/ODC, which is $9,423 more taxable income than the correct $67,794.08. That total results from adding the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income that already contains it (raising QBI to $3,310) and subtracting the repealed $1,796 DPAD."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It reached $67,813 of taxable income by deducting only the head's $108 IRA contribution and rounded wage and overtime amounts. The engine's IRA adjustment is $126.22, which includes Dependent 1's $18 contribution, and its overtime deduction is $13,899.29. That gives $67,794.08 taxable and a result $2.27 lower; its deductions and $3,200 of credits are otherwise identical."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the repealed $1,796 domestic production activities deduction (AGI $112,626 instead of $114,403.37). It also claimed $1,500 of other-dependent credit (three $500 credits), but only Dependent 1 and Dependent 2 qualify, for $1,000. Together these cut taxable income to $66,017 and added $500 of credit that does not exist."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round $13,500 with no computation. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions gives $67,794.08 taxable and $7,639.29 of tax, and $4,439.29 after $3,200 of CTC/ODC. $13,500 is the tax before credits on about $109,400 of taxable income, so it skipped the overtime deduction and the CTC/ODC and overstated income by tens of thousands."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset: a $17,000 standard deduction, $27,000 of personal exemptions, a 15% bracket and a $1,000 CTC phasing out above $110,000. OBBBA made TCJA permanent, so 2026 MFJ has a $32,200 standard deduction, no exemptions, 10%/12% brackets, a $2,200 CTC plus two $500 ODCs (phase-out at $400,000), and the $13,899 overtime deduction."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income that already includes it, which also inflated QBI to $3,310 (correct: 20% × $2,550 = $510). It also used a $30,700 standard deduction and a $24,150 top of the 10% bracket instead of 2026's $32,200 and $24,800. Taxable income came to $80,513 instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income (inflating QBI to $3,310), giving $79,013 taxable vs $67,794.08. It then added a $2,000 10% early-distribution additional tax, which is a Schedule 2 'other tax' and not part of income tax before refundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $21,208 of ESI and health premiums from wages, counted the $14,000 of SE partnership income twice, and deducted half of an SE tax that does not exist (the farm loss makes net SE earnings negative). It also treated the repealed $1,796 DPAD as QBI, used a $30,750 standard deduction, and left out the $13,899 overtime deduction. On credits it applied only a $2,000 CTC, missing the 2026 $2,200 amount and both $500 ODCs."
-us,scenario_064,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It added a $2,000 10% early-withdrawal additional tax on the 401(k) distribution, which is a Schedule 2 'other tax' and not part of income tax before refundable credits. It also brought AGI down to $97,723 by subtracting the overtime deduction plus an extra $2,800 net farm loss, and it replaced the $510 QBI deduction with the repealed $1,796 DPAD. That left $63,727 taxable instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income and ignored the farm loss when computing QBI ($6,670 vs $510). It deducted Dependent 1's $386 401(k) deferral although that dependent has no wages, took a half-SE-tax deduction, and left out the $13,899 qualified overtime deduction. It also denied the $500 ODC for the 18-year-old, used a $2,000 CTC, and reached $7,646 through unexplained adjustments."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used the full $97,295 of wages without removing the $2,315 traditional 401(k) deferral and deducted the repealed $1,796 DPAD instead of the $510 QBI deduction. Taxable income came to $68,842 instead of $67,794.08, making tax $125.71 too high."
-us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It answered $0 with no reasoning, but the liability is positive. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions leaves $67,794.08 taxable and $7,639.29 of tax, and the $3,200 of CTC/ODC offsets only part of it. The $0 answer treats deductions or credits as wiping out tax they do not cover."
-us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income twice (net business $16,550 instead of $2,550) and deducted half of an SE tax that does not arise because the farm loss makes net SE earnings negative. It also took the repealed $1,796 DPAD, inflated QBI to $3,310, and left out the $13,899 qualified overtime deduction, giving $89,947 taxable instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the entire $18,235 capital loss instead of the $3,000 annual limit and left out the $13,899 overtime deduction. It claimed no CTC/ODC on the false premise that income is above the thresholds, when the MFJ phase-out starts at $400,000. It then reported $14,629.40, exactly $7,000 more than its own $7,629.40 bracket computation."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It added a made-up $12,150 'child tax credit' to $4,564.81 of tax instead of subtracting the actual $2,200 CTC and $1,000 ODC. Its taxable income was also wrong: it subtracted $11,827 of retirement contributions, took both itemized medical expenses and a $30,000 standard deduction, and left out the $13,899 overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It assumed deductions and credits fully offset the tax without computing anything. After the $32,200 standard deduction, $13,899 overtime deduction and $510 QBI deduction, $67,812 of taxable income remains, which produces $7,641.46 of tax, and only $3,200 of CTC/ODC offsets it."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It treated liability as zero without any computation. Taxable income of $67,812 after all applicable deductions produces $7,641.46 of tax, which exceeds the $3,200 of nonrefundable CTC/ODC by $4,441.46."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,219 plus $3,200 of credits implies only ~$5,419 of tax, or ~$49,300 of taxable income. That is about $18,500 below the correct $67,812, so it took exclusions beyond the $2,315 401(k) deferral, $3,000 capital-loss limit, $108 IRA, $32,200 standard, $13,899 overtime and $510 QBI deductions; one example is treating the $19,648 employer-paid ESI premium as a reduction of reported wages."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,485 implies about $7,685 of pre-credit tax, so its taxable income was roughly $365 above $67,812.12. It understated a deduction relative to the $510 QBI deduction (20% of $19,350 partnership income net of the $16,800 farm loss) and the $108 IRA deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It subtracted only the $2,200 qualifying-child credit from roughly the correct ~$7,650 of tax. It omitted the $500 credit for other dependents for each of the disabled 27-year-old and the 18-year-old, which together take another $1,000 off."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its $92,978 AGI equals the correct $114,421 minus the $19,648 employer-paid ESI premium and the $1,796 DPAD. The given wages are not reduced by employer premiums, and the DPAD was repealed after 2017. This cut taxable income to $46,369 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $5,570 implies $77,217 of taxable income, which is $9,405 above the correct figure. That gap comes from adding the $14,000 SE partnership income on top of the $19,350 partnership total that already includes it, partly offset by deducting the repealed $1,796 DPAD and taking a $3,310 QBI deduction instead of $510."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $112,626 AGI is the correct $114,421 minus the repealed $1,796 DPAD. It also claimed $1,500 of other-dependent credits for three dependents when only the 27-year-old and the 18-year-old qualify, which is $1,000."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no derivation, and $13,500 is far above the $7,641.46 tax on $67,812 of taxable income. It ignored the $13,899 overtime deduction and the $3,200 of CTC/ODC that bring liability to $4,441.46."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It reached the correct $114,422 AGI but applied TCJA-sunset rules: a $17,000 standard deduction, $27,000 of personal exemptions, a 15% bracket and a $1,000 CTC phased out above $110,000. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, a $13,899 overtime deduction, 10%/12% brackets, a $2,200 CTC and two $500 ODCs with a $400,000 phaseout threshold."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership total that already includes it, which inflated QBI to $16,550 (a $3,310 deduction instead of $510). It also used a $30,700 standard deduction instead of $32,200, giving $80,513 of taxable income instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income, which is already inside the $19,350 partnership total, and took a $3,310 QBI deduction instead of $510, overstating taxable income at $79,013. It then added a $2,000 early-distribution penalty, which is not part of this income-tax-after-nonrefundable-credits measure."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 of employer and own health premiums from wages, double-counted the $14,000 SE partnership income, took a half-SE-tax deduction despite negative net SE earnings, and treated the repealed $1,796 DPAD as QBI. It also omitted the $13,899 overtime deduction and the $1,000 of ODC, and used a $2,000 CTC instead of $2,200."
+us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"It added a $2,000 10% early-withdrawal penalty on the 401(k) distribution, which is not included in this measure. It also understated AGI at $97,723 and deducted the repealed $1,796 DPAD instead of the $510 QBI deduction. The correct result is $7,641.46 of tax on $67,812 minus $3,200 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income as separate from the $19,350 partnership total, took QBI on $33,350 without netting the $16,800 farm loss ($6,670 instead of $510), and deducted the dependent's own 401(k) and IRA contributions on the parents' return. It also omitted the $13,899 overtime deduction and denied the $500 ODC to the 18-year-old. Its final $7,646 is an unexplained adjustment from its own $7,815."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It did not exclude the head's $2,315 traditional 401(k) deferral from wages (AGI $116,737 instead of $114,421). It also deducted the repealed $1,796 DPAD in place of the $510 QBI deduction (20% of $19,350 − $16,800), which overstated taxable income by about $1,030."
+us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It answered $0 with no reasoning. Taxable income of $67,812 after the standard, overtime and QBI deductions produces $7,641.46 of tax, and the $3,200 of CTC/ODC leaves $4,441.46."
+us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership total, which created $16,550 of business income, a phantom half-SE-tax deduction and a $3,310 QBI deduction. It also deducted the repealed $1,796 DPAD and omitted the $13,899 overtime deduction, leaving $89,947 of taxable income instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $18,235 capital loss instead of the $3,000 limit and omitted the $13,899 overtime deduction. It wrongly said the CTC/ODC were unavailable even though the MFJ phaseout starts at $400,000. It then reported $14,629.40 even though its own computed tax was $7,629.40."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It deducted the full $18,235 capital loss, took both itemized medical expenses and the standard deduction, and invented a $12,150 child tax credit. It then added that credit to tax instead of subtracting it. The correct credits are $2,200 CTC plus $1,000 ODC, subtracted from $7,641.46 of tax."
us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It equated the refundable credit with the full $2,000 CTC for Child 1, skipping the ACTC computation entirely — the CTC first offsets federal income tax liability and only the unused remainder becomes refundable, capped at $1,700 per child and at 15% of earned income over $2,500. With $97,295 of wages plus $20,000 of taxable 401(k) distributions and $19,350 of partnership income, MFJ tax before refundable credits far exceeds $2,000, so the entire CTC is consumed nonrefundably and nothing spills into the refundable column. Its own AGI estimate of ~$120,610 also double-counted, adding partnership income while ignoring that the capital loss deduction is capped at $3,000."
us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning reached the correct conclusion — 'with significant federal tax owed, CTC is nonrefundable' — and it then submitted the $1,700 statutory ACTC ceiling anyway, converting a per-child maximum into an entitlement. The ACTC equals only the CTC remaining after the credit is applied against federal income tax liability; here liability on ~$98,810 of income absorbs the full $2,000 for the one under-17 child, so the refundable amount is $0."
us,scenario_064,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It counted Dependent 2 (age 18) as a CTC qualifying child, but the credit requires age under 17 at year end, so that dependent yields only the $500 nonrefundable other-dependent credit rather than a $1,700 refundable share. It then paid out the per-child ACTC ceiling without applying the rule that only the CTC unused against tax liability is refundable; MFJ tax before refundable credits on ~$98,810 of income exceeds $2,000, so even the single qualifying child's credit is fully nonrefundable and the correct total is $0. Checking the $400,000 phaseout threshold was irrelevant — the binding constraint is the liability offset, not the income phaseout."
@@ -5255,23 +5349,24 @@ us,scenario_064,head_medicare_eligible,glm-5.3,parse_contract_failure,missing_ou
us,scenario_064,head_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model computed federal employee FICA exactly right ($6,032.29 + $1,410.78 = $7,443.07) and then invented a nonexistent Wisconsin employee payroll tax at ""approximately 3%"" ($2,918.85); Wisconsin imposes no mandatory employee-side state payroll contribution, so that entire component is fabricated. It then compounded the error by submitting $12,318.64 after computing a stated total of $10,361.92, adding $1,956.72 under an unexplained ""adjusting for precise Wisconsin rate"" step."
-us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model's derivation is fully correct and its explanation states the total as $7,443.07, matching the reference exactly, but it submitted 7407.36 as the numeric value. The failure is a transcription error between the stated total and the submitted field, not a substantive rule or base error; its only substantive slips (quoting a ~$176,100 wage base) are immaterial because wages fall far below any cap."
-us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model reduced the FICA wage base by the $1,560 ""health insurance premiums excluding Medicare Part B"" input, computing tax on $95,735 instead of $97,295. That input is individually paid premium expense, not a Section 125 cafeteria-plan salary reduction, so it does not shrink Social Security or Medicare wages; the $119.34 shortfall is exactly 7.65% of $1,560."
-us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model stripped both the $19,648 employer-sponsored insurance premium and the $2,315 traditional 401(k) deferral from wages, taxing $75,332 instead of $97,295. Elective 401(k) deferrals are expressly included in FICA wages under IRC §3121(v)(1)(A), and the ESI premium amount does not reduce the modeled Social Security/Medicare base, so both subtractions are invalid."
-us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $19,648 employer-sponsored insurance premium from gross wages and applied 7.65% to $77,647, producing $5,940. The ESI premium input does not reduce the FICA wage base here; the correct base is the full $97,295, and the $1,503.07 shortfall is precisely 7.65% of the $19,648 it removed."
-us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model declared $77,647 the ""FICA taxable wages"" by treating the $19,648 employer-sponsored insurance premium as a pre-tax salary reduction, then applied 6.2% and 1.45% to that reduced base. The FICA base is the head's full $97,295 of gross wages, so both components are understated by 7.65% of $19,648."
-us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model applied Social Security and Medicare to wages net of ""pre-tax health insurance deductions,"" landing on the same $5,940 that 7.65% of $97,295 − $19,648 produces. The $19,648 employer-sponsored insurance premium does not reduce the Social Security or Medicare wage base in this computation; tax is owed on the full $97,295."
-us,scenario_064,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model explicitly netted the $19,648 employer-sponsored insurance premium out of wages to get a $77,647 base before applying 6.2% and 1.45%. That premium is not a FICA-exempt salary reduction here, so the correct base is $97,295 and the answer is short by exactly 7.65% × $19,648 = $1,503.07."
-us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key never reached the outputs object. This is a contract failure in emitting the submit_outputs response rather than a substantive misapplication of the FICA rules."
-us,scenario_064,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"The response contained no payroll_tax value and no accompanying explanation, leaving the requested key absent from the outputs object. The failure is in producing a parseable submission, not in the payroll tax computation itself."
-us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model gave no derivation and submitted $12,306, which exceeds the correct wage-only employee FICA of $7,443.07 by $4,862.93 — an excess that can only come from taxing non-wage income or employer-side liability, since 7.65% of any wage figure the household has cannot reach $12,306. That excess corresponds to layering the excluded 15.3% self-employment tax onto the head's $33,350 of partnership and self-employment income, which the output definition explicitly removes from payroll_tax."
-us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model gave no computation and submitted $8,659, which is 7.65% of a roughly $113,000 wage base rather than of the head's actual $97,295. That inflated base reflects adding the separately listed $13,899 FLSA overtime premium on top of gross wages, even though the prompt states gross wages are annual totals already including overtime pay; the correct answer is 7.65% × $97,295 = $7,443.07."
-us,scenario_064,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model applied 7.65% to $77,647 after removing the $19,648 employer-sponsored insurance premium as a pre-tax deduction. Employee Social Security and Medicare are owed on the head's full $97,295 of gross wages, making the answer short by 7.65% of the premium it excluded."
-us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so the key is missing from the submitted outputs. The failure is in delivering a parseable answer, not in the underlying FICA reasoning."
-us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model submitted $0 with no reasoning, which is the answer only if the household has zero FICA-covered wages. The head has $97,295 of gross wages and salaries, which carry 6.2% Social Security and 1.45% Medicare employee tax for a total of $7,443.07; the model zeroed out the wage base entirely."
-us,scenario_064,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"The model subtracted the $2,315 traditional 401(k) contribution as ""pre-tax"" and applied 6.2% and 1.45% to $94,980. Elective 401(k) deferrals reduce income tax but remain wages for Social Security and Medicare under IRC §3121(v)(1)(A), so the base is $97,295 and the answer is short by 7.65% × $2,315 = $177.10 (plus rounding)."
-us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly identified the $97,295 wage base, correctly computed Social Security of $6,032.29 and Medicare of $1,410.78, and correctly ruled out Additional Medicare Tax — components that sum to $7,443.07 — yet submitted 8529.65. The submitted value contradicts its own itemized components by $1,086.58 with no stated basis, so the failure is in the final total it reported rather than in any rule it applied."
+us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the FICA pieces correctly ($6,032.29 + $1,410.78), then added a made-up 3% Wisconsin employee payroll tax ($2,918.85). Wisconsin has no mandatory employee state payroll tax. It then inflated the total again, without explanation, to $12,318.64. The correct answer is FICA alone: $7,443.07."
+us,scenario_064,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"It declined to calculate employee Social Security and Medicare tax and submitted 0, even though the head has $97,295 of wages. Those wages produce $6,032.29 of Social Security tax plus $1,410.78 of Medicare tax, for $7,443.07."
+us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its reasoning reached the correct total: 6.2% and 1.45% on $97,295, giving $7,443.07. It then submitted the unrelated figure $7,407.36 and called it rounding. This was a transcription error in the final step, not an error in the payroll-tax rules."
+us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the head's $1,560 health insurance premiums from wages and used $95,735 as the FICA base. Nothing in the facts makes those premiums a pre-tax payroll deduction, so the full $97,295 is subject to FICA, giving $7,443.07."
+us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It cut the FICA base to $75,332 by subtracting both the $19,648 ESI premium and the $2,315 traditional 401(k) deferral. Traditional 401(k) elective deferrals stay in FICA wages under IRC 3121(v)(1)(A), and the ESI premium is not a listed salary reduction. FICA applies to all $97,295 of wages."
+us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It assumed the $19,648 employer-sponsored insurance premium was a pre-tax cafeteria-plan deduction and taxed only $77,647. The facts list gross wages of $97,295 with no salary reduction, so 7.65% applies to $97,295, giving $7,443.07."
+us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $19,648 ESI premium as a pre-tax deduction from FICA wages, which left a base of $77,647 and a total of $5,940. The employee FICA base is the full $97,295 of listed gross wages, which gives $6,032.29 + $1,410.78 = $7,443.07."
+us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It subtracted pre-tax health insurance (the $19,648 ESI premium) from wages before applying FICA, which produced $5,940. No salary reduction is listed, so Social Security and Medicare apply to the full $97,295 of wages, giving $7,443.07."
+us,scenario_064,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages to $77,647 by treating the $19,648 ESI premium as a pre-tax health deduction. The correct base is the full $97,295 of gross wages: 7.65% of that is $7,443.07."
+us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is 7.65% of the head's $97,295 wages, or $7,443.07."
+us,scenario_064,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is $6,032.29 of Social Security tax plus $1,410.78 of Medicare tax, or $7,443.07."
+us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"At 7.65%, its $12,306 implies a FICA base of about $160,900, far above the head's $97,295 of wages. It folded non-wage income into the base: partnership/S-corp income, self-employment income, and 401(k) distributions are not employee FICA wages. Only the $97,295 is taxable, giving $7,443.07."
+us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"At 7.65%, its $8,659 implies a wage base of about $113,200 instead of the $97,295 of listed gross wages. It added income on top of wages, such as the $13,899 FLSA overtime premium, which the listed gross wages already include. Employee FICA on $97,295 is $7,443.07."
+us,scenario_064,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 7.65% to $77,647 after treating the $19,648 employer-sponsored insurance premium as a pretax salary reduction. No such reduction is listed, so the FICA base is the full $97,295, giving $7,443.07."
+us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is 7.65% of the head's $97,295 wages, or $7,443.07."
+us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It submitted 0 with no reasoning, which ignores the head's $97,295 of wages. Those wages carry $6,032.29 of employee Social Security tax and $1,410.78 of employee Medicare tax, for $7,443.07."
+us,scenario_064,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"It subtracted the $2,315 traditional 401(k) deferral and taxed only $94,980. Under IRC 3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base is $97,295 and the total is $7,443.07."
+us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"Its reasoning computed the correct pieces, $6,032.29 of Social Security tax and $1,410.78 of Medicare tax, which add to $7,443.07. It then submitted the unrelated figure $8,529.65. The error was in adding up and reporting the final number, not in the payroll-tax rules."
us,scenario_064,reduced_price_school_meals_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It built the base by adding the full $19,350 partnership/S-corp share to the $14,000 self-employment partnership earnings, double-counting the same K-1 income and treating non-SE distributive share as SE earnings, so its $16,550 base stayed positive instead of the non-positive net earnings the -$16,800 farm loss produces. It then halved its own $2,337.65 result on the theory that only 50% is owed, confusing the one-half-of-SE-tax above-the-line income tax deduction under section 164(f) with a reduction of the SE tax itself. The submitted $1,965.48 matches neither of its two computations and is an unexplained fudge."
us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"It correctly identified that the -$16,800 farm loss offsets the $14,000 partnership self-employment earnings and zeroes the base, then overrode that with an invented claim that PolicyEngine taxes the $14,000 in isolation. Section 1402(a) net earnings aggregate all of a person's trades and businesses, so the Schedule F farm loss is netted against the partnership SE earnings and leaves nothing to tax."
@@ -5310,140 +5405,144 @@ us,scenario_064,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model
us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled, applied the one-third reduction for living in another person's household ($644.67 × 12 ≈ $7,736) and added an estimated Wisconsin supplement to reach $7,952. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), so no federal benefit or state supplement is paid and SSI is $0."
us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SSI value and no explanation, so there is no answer to score. The correct derivation is $0: no member meets SSI's aged/blind/disabled requirement under the engine's inputs."
us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model correctly excluded Head and Spouse as neither 65+ nor disabled, but wrongly declared Dependent 1 'categorically SSI-eligible' because of the disability flag and paid $994 × 12 = $11,928. The engine's SSI aged/blind/disabled test is also False for Dependent 1, so no member is eligible and SSI is $0."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It first overstated WI AGI at $118,248, adding a $2,500 capital-loss add-back and wrongly deducting about $989 of half-SE tax even though the -$16,800 farm loss wipes out net SE earnings. That led to about $4,964 of gross tax. It then cut this to $2,699 with invented nonrefundable credits (an itemized-deduction credit plus unexplained 'precision' adjustments), although no nonrefundable credit applies. The engine's result is $4,606, on $102,144 of taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 self-employment partnership income a second time on top of the $19,350 partnership/S-corp income, which already includes it, and also subtracted the $1,796 domestic production deduction. That gave a federal AGI of $126,626. It then added a $2,500 capital-loss add-back, so WI taxable income came out near $117,700 instead of the engine's $102,144."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from a guessed federal taxable income instead of WI AGI minus the $8,759 phased standard deduction and $3,500 of exemptions. It also described Wisconsin's rates as 5.3%–7.65%, when this income is actually taxed at 3.5%/4.4%/5.3%. As a result, $4,512 does not come from applying the 2026 schedule to $102,144."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It claimed only 4 exemptions ($2,800) instead of 5 ($3,500), and it started the 5.3% rate at $36,840 instead of at the top of the widened 4.4% bracket (about $50,500). It then subtracted about $875 for a school property tax credit and an itemized deduction credit. The household reports no property tax or rent, and its itemized deductions are below the standard deduction, so neither credit exists."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It computed $4,784 on about $100,000 of taxable income using a 4.4% bracket that ends at $38,190. It then raised the figure to $5,074 while saying it was 'reducing' tax for exemptions, which contradicts its own arithmetic. The engine applies 4.4% up to about $50,500 on $102,144 of taxable income, giving $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round-number estimate of $4,600 instead of running the brackets on the exact taxable income. The correct figure is $114,403 WI AGI minus the $8,759 phased standard deduction and $3,500 of exemptions, or $102,144, which produces $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a $2,500 capital-loss add-back, bringing WI AGI to $116,922, which is $2,519 above the engine's $114,403. It also used a standard deduction of about $8,000 instead of $8,759. Together these raised taxable income to $105,422 instead of $102,144, which pushed the tax up to $4,802."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It modeled the sliding-scale standard deduction with an invented $22,010 base and a phase-out ending at $134,000, which left only $3,957 instead of the engine's $8,759. It also added $2,500 to AGI for capital losses and started the 5.3% rate at $36,840, producing $5,257 of gross tax. It then subtracted an invented $520 of nonrefundable credits, although none apply."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income a second time on top of the $19,350 partnership total and subtracted the $1,796 domestic production deduction, reaching $126,626 of AGI. It then used only 4 exemptions and an overstated $14,000 standard deduction. Its stated $109,826 of taxable income would produce more than $5,000 of tax, so its final $4,200 does not follow from its own inputs."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It put WI taxable income at about $125,000, far above the engine's $102,144, and claimed only three $700 dependent exemptions, leaving out the two filer exemptions. Applying the rates to that inflated base produced $6,200 instead of $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used an invented schedule of 4% on the first $23,000 and 5.3% on everything above that. The actual rates are 3.5%, then 4.4% up to about $50,500, then 5.3%. It also left out the $3,500 of personal exemptions, which pushed the tax to $5,003."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $25,000 MFJ standard deduction without applying Wisconsin's sliding-scale phase-out, which cuts it to $8,759 at this income. It also subtracted the $1,796 domestic production deduction from WI AGI. Together these understated taxable income at $82,566 instead of $102,144."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its WI taxable income of $119,775 is about $17,600 above the engine's $102,144. That means it included income the engine does not count, such as the SE partnership income counted a second time, or it failed to net the farm loss. Taxing that inflated base gave $5,830."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It treated the 5 × $700 exemptions as a $3,500 credit subtracted from tax instead of a $3,500 deduction from income, which removed about $3,300 too much. It also used the outdated 3.54%/4.65% rates and an overstated $13,197 standard deduction."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It used a sliding-scale standard deduction of $13,506 instead of the engine's $8,759, which cut taxable income to $97,576 instead of $102,144. It then subtracted a $150 'dependent credit' that Wisconsin does not provide as a nonrefundable credit here."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,652 does not match the 2026 MFJ schedule applied to $102,144 of taxable income ($114,403 AGI − $8,759 standard deduction − $3,500 exemptions). Its number is consistent with slightly overstated taxable income or a narrower 4.4% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its starting AGI of $92,978 leaves out about $21,400 of income that the engine counts; the engine's WI AGI is $114,403. The largest omission is the $20,000 taxable 401(k) distribution. As a result, taxable income fell to $78,969 instead of $102,144."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It stated taxable income of $82,256, which means it took a standard deduction far larger than the phased $8,759. Its $4,356 of tax is also higher than what the 2026 schedule yields on $82,256, so both its deduction and its bracket math are wrong."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $6,950 overshoots the correct $4,606 by $2,344. That amount corresponds to taxing about $44,000 more than the correct $102,144 of taxable income, or to ignoring the phased standard deduction and exemptions."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,125 is $481 below the tax the 2026 MFJ schedule yields on $102,144. That gap corresponds to about $9,000 of overstated deductions or phantom nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied unspecified nonrefundable credits, but none apply to this household: the spouse has no earnings, and itemized deductions are below the $8,759 standard deduction. Its $3,212 is $1,394 below the $4,606 that the schedule yields on $102,144 of taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted nonrefundable credits that do not apply to this household and/or overstated the standard deduction. The engine's standard deduction is phased to $8,759, taxable income is $102,144, and the tax is $4,606, not $2,490."
-us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference."
-us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and retirement adjustments eliminate Wisconsin tax. In fact, WI AGI is $114,403, the phased standard deduction is only $8,759, and exemptions are $3,500, which leaves $102,144 taxable and $4,606 of tax."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It assumed that deductions and credits wipe out the tax, but the household has $102,144 of Wisconsin taxable income after the $8,759 phased standard deduction and $3,500 of exemptions. No nonrefundable credit applies, so the tax is $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It overstated the phased MFJ standard deduction at $15,166 instead of $8,759 and claimed only two exemptions instead of five. It then subtracted a $472 itemized-deduction credit. That credit is zero because allowable itemized deductions (medical costs above 7.5% of AGI, with no mortgage interest input) are below the standard deduction."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $2,800 is $1,806 below the tax on $102,144 of taxable income, which means it applied phantom nonrefundable credits or a standard deduction far larger than the phased $8,759."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,872 is $266 above the correct tax on $102,144, which fits starting the 5.3% rate near $39,000–$40,000 instead of at the top of the widened 2026 4.4% bracket (about $50,500), combined with slightly overstated taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It applied 'exemption credits' against tax. Wisconsin's $700 × 5 exemptions are a $3,500 reduction of income, not a credit, so treating them as tax credits cut the result to $4,071 instead of $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It adjusted AGI with a $2,500 capital-loss add-back and a $2,340 health-insurance subtraction, then applied a sliding standard deduction and brackets that produced only $3,254. That is roughly $1,350 below the tax the 2026 schedule yields on the correct $102,144 of taxable income, so its deduction or bracket figures were far off the engine's $8,759 deduction and ~$50,500 top of the 4.4% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $5,654 is about $1,050 above the tax on $102,144 of taxable income, which fits leaving out most of the phased standard deduction and the exemptions, or taxing at 5.3% well below the ~$50,500 top of the 4.4% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its starting AGI (about $114,422) was close, but its final $4,200 implies deductions about $7,700 larger than the engine's $8,759 phased standard deduction plus $3,500 of exemptions. It also referred only to dependent exemptions, not to all five."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a health-premium deduction of about $4,100, bringing income to $110,286, whereas the engine's WI AGI is $114,403. It also used a $10,208 sliding standard deduction instead of $8,759. Both steps understated taxable income, which led to $4,472."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded $4,500 without computing the tax. The correct taxable income is $114,403 − $8,759 − $3,500 = $102,144, and the 2026 MFJ schedule gives $4,606 on that amount."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It understated the phased standard deduction at about $6,500 instead of $8,759 and never subtracted the $3,500 of personal exemptions. That left taxable income of about $107,900 instead of $102,144, which inflated the tax to $4,900."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $114,523 and a $9,076 standard deduction instead of $114,403 and $8,759. It also used invented brackets: 3.5% up to $29,350 and 4.4% up to $58,700, instead of 3.5% up to roughly $20,000 and 4.4% up to about $50,500. These errors partly cancel and leave the answer $4 off, at $4,610."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $102,274 is within $130 of the engine's $102,144, yet it computed $4,876, which is $270 too high. So it taxed far more income at 5.3% than the 2026 schedule does, where the 4.4% bracket for joint filers extends to about $50,500."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied invented 4.75%/5.75% rates instead of the actual 4.4%/5.3%. It also started from an understated AGI of $106,225, used an overstated $13,500 standard deduction, and left out the $3,500 of personal exemptions."
-us,scenario_064,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It treated the federal overtime deduction as having reduced federal AGI to $97,723 and then added it back. It still ended up with about $90,000 of taxable income, well below the engine's $102,144, because it overstated the deductions taken from the engine's $114,403 WI AGI."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 6.27% rate to its third bracket, but Wisconsin's third-bracket rate is 5.3%. It also understated the phased standard deduction at $5,481 instead of $8,759, which inflated the tax to $6,091."
-us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned $0 with no reasoning, even though the household has $102,144 of Wisconsin taxable income after the $8,759 phased standard deduction and $3,500 of exemptions, which produces $4,606 of tax."
-us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It took a $31,300 MFJ standard deduction without applying Wisconsin's sliding-scale phase-out, which reduces it to $8,759 at this income, and it left out the exemptions. It also overstated AGI at $127,253 and used invented bracket thresholds of $29,430 and $58,860."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the overtime premium out of wages, reducing them to $83,396, even though overtime stays in Wisconsin AGI. It also used an unphased standard deduction of about $13,500, and its final $4,221.67 does not follow from its own $3,767 bracket calculation."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $11,827 of retirement contributions a second time and also deducted medical expenses directly from income. The 401(k) deferral is already excluded from wages, and Wisconsin has no such medical deduction when the standard deduction is used. Both errors pushed taxable income far below $102,144."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It added back $2,500 on the belief that Wisconsin caps net capital losses at $500, but Wisconsin uses the same $3,000 joint limit. It also deducted half of an SE tax that doesn't exist, because the $16,800 farm loss wipes out the $14,000 SE partnership income. After taxing at the old schedule where 5.3% starts at $38,190 (giving $4,964), it cut the result to $2,699 using an itemized-deduction credit and unexplained adjustments, even though no nonrefundable credits apply."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its $126,626 federal AGI counts the $14,000 SE partnership income a second time, even though it is already inside the $19,350 partnership/S-corp income, and wrongly subtracts the $1,796 domestic production deduction. It then added $2,500 more under a nonexistent $500 Wisconsin capital-loss cap. That inflated taxable income to about $117,700 instead of $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It started from 'federal taxable income' that subtracts the full $18,235 capital loss instead of the $3,000 limit, and it cited a made-up 5.3%–7.65% Wisconsin rate range. It never applied the $8,953.27 sliding standard deduction and five $700 exemptions to the $114,421.41 Wisconsin AGI, so $4,512 is an unsupported guess."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It explicitly skipped the Wisconsin computation and reported $0. With $114,421.41 of Wisconsin AGI, the $8,953.27 sliding standard deduction and $3,500 of exemptions, this joint return has $101,968.14 taxable and owes $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It left out the $2,315 401(k) wage exclusion and the $108 IRA deduction (AGI $116,845 instead of $114,421.41), and it counted four $700 exemptions instead of five. It used the outdated schedule where 5.3% starts at $36,840, not the expanded 4.4% bracket running to roughly $69,000. It then subtracted about $875 of school property tax and itemized deduction credits, which don't apply: no property tax or rent is listed and the return uses the standard deduction."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied the pre-2025 joint brackets, where 5.3% starts at $38,190, instead of the 2026 schedule where 4.4% runs to roughly $69,000, and got $4,784. It then raised the answer to $5,074 while claiming exemptions and the standard deduction reduce tax, so the final number contradicts its own computation."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It added back $2,500 on the assumption of a $500 Wisconsin capital-loss limit. Wisconsin allows $3,000 on a joint return, so Wisconsin AGI equals the federal $114,421.41. It then taxed the inflated $105,422 using the old schedule where 5.3% starts near $38,000, not the expanded 4.4% bracket to roughly $69,000."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It deducted a $990 half-SE-tax that doesn't exist, because the farm loss offsets the SE partnership income. It also disallowed the $108 IRA deduction, added back $2,500 under a nonexistent $500 capital-loss cap, and used a made-up standard-deduction phase-out ($22,010 phased out linearly by $134,000), which gave $3,957 instead of $8,953.27. It then taxed at the old brackets and subtracted $520 of invented credits."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its $126,626 AGI counts the $14,000 SE partnership income twice, even though it is already part of the $19,350 partnership/S-corp income, and subtracts the $1,796 domestic production deduction, which is not an AGI adjustment. It also used about $14,000 of standard deduction and only four exemptions, and its $4,200 doesn't follow from its own $109,826 taxable income at the rates it named."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It put Wisconsin taxable income at about $125,000 instead of $101,968.14, which means it overstated income by about $23,000. It also allowed only three $700 dependent exemptions, leaving out the head's and spouse's exemptions (five in total), and taxed the excess at 5.3%."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up two-rate schedule (4% on the first $23,000 and 5.3% on the rest) instead of Wisconsin's 3.5%/4.4%/5.3% brackets, where 4.4% runs to roughly $69,000 for joint filers. It also subtracted only the standard deduction and skipped the $3,500 of personal exemptions."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It took a flat $25,000 standard deduction, ignoring Wisconsin's sliding-scale phase-down, which cuts the joint deduction to $8,953.27 at $114,421.41 of income. It also subtracted the $1,796 domestic production deduction from Wisconsin AGI. Its taxable income of $82,566 is about $19,400 too low."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its Wisconsin taxable income of $119,774.69 is $17,806 above the correct $101,968.14, a gap about the size of the $16,800 farm loss, which it did not subtract from income. Taxing that excess at 5.3% produced the $1,230 overstatement."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It treated the five $700 personal exemptions as a $3,500 credit against tax instead of a $3,500 deduction from income, which removed $3,500 directly from the liability. It also used the outdated 3.54%/4.65% rates and an overstated $13,197.25 standard deduction instead of $8,953.27."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It overstated the sliding-scale standard deduction ($13,506 instead of $8,953.27). It used the pre-2025 brackets, where 5.3% starts at $38,190, instead of the 2026 schedule with 4.4% to roughly $69,000. It then subtracted a $150 'dependent credit' that Wisconsin does not have."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is $114,421.41 of Wisconsin AGI, minus the $8,953.27 sliding standard deduction and $3,500 of exemptions, giving $101,968.14 taxable and $4,599.62 of tax. Its $4,652.12 is $52.50 higher, which matches about $990 of extra taxable income at the 5.3% marginal rate, consistent with phasing the standard deduction down too far."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from a $92,978 AGI, $21,443 below the correct $114,421.41. That left taxable income at $78,969 instead of $101,968.14, so the tax came out about $930 too low."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $82,256 of Wisconsin taxable income is about $19,700 below the correct $101,968.14, so it understated income or overstated deductions well beyond the $8,953.27 sliding standard deduction and $3,500 of exemptions. Its $4,356 also doesn't follow from that base under the 3.5%/4.4%/5.3% schedule."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $6,950 is $2,350 above the correct $4,599.62, which requires about $44,000 more taxable income than the correct $101,968.14 at the 5.3% marginal rate. It therefore failed to subtract the farm loss, the $3,000 capital loss, the $8,953.27 sliding standard deduction and the $3,500 of exemptions from Wisconsin income."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,125 is $475 below the correct $4,599.62, which equals about $9,000 of extra deductions at the 5.3% marginal rate. That is consistent with an under-phased standard deduction of about $18,000 instead of the $8,953.27 the sliding scale allows at $114,421.41 of income."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,212 is $1,388 below the correct $4,599.62, which equals about $26,000 of excess deductions at 5.3%. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply, with no nonrefundable credits."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,490 is $2,110 below the correct $4,599.62, which equals about $40,000 of excess deductions or invented credits. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions reduce the $114,421.41 Wisconsin AGI, and no nonrefundable credits apply."
+us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions and retirement adjustments reduce Wisconsin tax to zero. In fact the sliding standard deduction ($8,953.27) and five exemptions ($3,500) total only $12,453.27 against $114,421.41 of Wisconsin AGI, leaving $101,968.14 taxable."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits fully offset Wisconsin tax. In fact only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply against $114,421.41 of Wisconsin AGI, and no nonrefundable credits apply, so $4,599.62 is owed."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It left out the $3,000 capital-loss deduction (using $117,222 of income instead of $114,421.41). It overstated the standard deduction at $15,166 instead of $8,953.27 and allowed only two $700 exemptions instead of five. It also subtracted a $472 itemized-deduction credit, which is zero here because qualifying medical costs don't clear the 7.5%-of-AGI floor and no mortgage interest is listed."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,800 is about $1,800 below the correct $4,599.62, which equals about $34,000 of excess deductions or invented credits. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply to the $114,421.41 Wisconsin AGI, with no nonrefundable credits."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,872 matches the pre-2025 joint brackets, where 5.3% starts near $38,190; those brackets give about $4,889 on $101,968.14. Under the 2026 schedule the 4.4% rate runs to roughly $69,000, which brings the tax to $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated Wisconsin's personal exemptions as 'exemption credits' instead of $700 deductions from income for each of the five people. Its $4,071 is $528.62 below the correct $4,599.62, which equals about $10,000 of excess deductions at the 5.3% marginal rate."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It added back $2,500 under an outdated $500 Wisconsin capital-loss cap, but Wisconsin uses the $3,000 joint limit. It also took a $2,340 medical-care-insurance subtraction, which does not apply because the head and dependent have employer-sponsored coverage where the employer pays part of the premium. Its 'estimated indexed' brackets then put the tax about $1,345 too low on a taxable base near $102,000."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $5,654 is $1,054 above the correct $4,599.62, which equals about $20,000 more taxable income than $101,968.14 at 5.3%. That is consistent with leaving out the $8,953.27 sliding standard deduction and $3,500 of exemptions and using the pre-2025 brackets."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It started from the correct $114,421.41 AGI, but its $4,200 is about $400 too low, which equals roughly $7,500 of extra deductions at 5.3%. It used a standard deduction well above the $8,953.27 that the sliding scale allows at this income."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted a $4,136 health-premium deduction to reach $110,286, but Wisconsin's medical care insurance subtraction does not apply to employer-sponsored coverage where the employer pays part of the premium, so Wisconsin AGI stays at $114,421.41. The lower base also pushed the sliding standard deduction up to $10,207.94 instead of $8,953.27."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,500 is $99.62 below the correct $4,599.62, which equals about $1,880 of excess deductions at 5.3%. That is consistent with a standard deduction above the $8,953.27 sliding-scale amount at $114,421.41 of income."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It left out the five $700 personal exemptions ($3,500) entirely, computing taxable income as $114,422 minus only the standard deduction. It also understated the sliding standard deduction at about $6,500 instead of $8,953.27, leaving taxable income about $5,900 too high."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its standard deduction, exemptions and taxable income ($101,947) were essentially right, but it used made-up equal-width brackets of $29,350 each. That made the 3.5% bracket too wide and ended the 4.4% bracket at $58,700 instead of the 2026 joint threshold of roughly $69,000, producing $4,610 instead of $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its taxable income of about $102,274 is close, but the $4,876 comes from the pre-2025 brackets, where 5.3% starts near $38,000–$39,000. The 2026 joint schedule applies 4.4% up to roughly $69,000, which brings the tax down to $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $106,225 AGI (about $8,200 too low), an unreduced $13,500 standard deduction, and no personal exemptions. It also applied made-up 4.75%/5.75% rates on 2024 thresholds instead of Wisconsin's 3.5%/4.4%/5.3% schedule."
+us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started from a $97,723 federal AGI and added back $13,899 for the federal overtime deduction. That deduction is taken below AGI, so federal AGI already includes all wages and equals $114,421.41. Getting from $111,622 down to about $90,000 of taxable income took about $21,600 of deductions, versus the correct $12,453.27 ($8,953.27 sliding standard deduction plus $3,500 of exemptions)."
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied a 6.27% third-bracket rate that doesn't exist; Wisconsin's rate is 5.3%. Its $116,737 AGI also leaves out the $2,315 traditional 401(k) wage exclusion, and its $5,481 sliding standard deduction is too small compared with $8,953.27."
+us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It reported $0 with no derivation. With $114,421.41 of Wisconsin AGI, the $8,953.27 sliding standard deduction and $3,500 of exemptions, taxable income is $101,968.14 and the tax is $4,599.62, with no nonrefundable credits to offset it."
+us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It used roughly the federal joint standard deduction ($31,300) instead of Wisconsin's sliding-scale deduction, which phases down to $8,953.27, and it left out the $3,500 of exemptions. Its $127,253 AGI is about $12,800 too high, and its $29,430/$58,860 bracket thresholds are wrong. These errors partly offset each other, giving $4,291."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It removed $13,899 of FLSA overtime from wages when computing AGI, but the federal overtime deduction is taken below AGI and Wisconsin does not adopt it, so those wages stay in Wisconsin AGI. It also used an unreduced $13,500 standard deduction, left out the $3,500 of exemptions, used a 4.65% rate, and reported $4,221.67 even though its own arithmetic gave $3,767."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted $11,827 of 'retirement contributions' after AGI, but the $2,315 traditional 401(k) deferral is already excluded from wages and Roth contributions are never deductible. It also deducted medical expenses, which Wisconsin does not allow as a deduction from income. Its $101,610 AGI starting point is already about $12,800 too low."
us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model enumerated every Wisconsin refundable credit — Homestead (income too high), the WI EITC (federal EITC is $0, and the WI credit is a straight percentage of it), Farmland Preservation, and the Veterans and Surviving Spouses Property Tax Credit — and stated that each one does not apply, then submitted $154 rather than the $0 its own enumeration produces. No credit it named yields $154: the Homestead Credit is additionally barred because no rent or property tax is listed, and the Married Couple and Itemized Deduction credits it correctly identified as nonrefundable are excluded from this output entirely."
us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the state_refundable_credits key from its outputs object, so it submitted no value for a variable the prompt required exactly once. The correct derivation sums Wisconsin's refundable credits — WI EITC at $0 because the federal EITC is $0 on $97,295 of wages, Homestead at $0 under its household-income ceiling and with no rent or property tax listed, Farmland Preservation at $0 absent a preservation agreement — for a total of $0."
us,scenario_064,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no state_refundable_credits entry, leaving the required key unanswered rather than making a substantive Wisconsin credit error. Working the rule set through yields $0: the WI EITC is a percentage of a federal EITC that is $0 at this income, the Homestead Credit fails both the income ceiling and the requirement that rent or property tax be paid, and the Farmland Preservation Credit requires an unlisted farmland preservation agreement."
us,scenario_064,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model produced no value or explanation for state_refundable_credits, so the answer failed the output contract rather than the tax rules. Wisconsin's refundable credits all evaluate to $0 for this household — the WI EITC tracks a federal EITC of $0 at $97,295 of wages, the Homestead Credit is blocked by its income ceiling and by the absence of rent or property tax, and the Farmland Preservation and Veterans property tax credits require unlisted qualifying facts."
us,scenario_064,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_066,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a minimum-earnings requirement — ""EITC which requires substantial earned income"" — when the childless EITC phase-in credits 7.65% of the very first dollar of earnings with no floor. Applying the actual phase-in rate to the stated $520 of wages gives $39.78; the model's $0 is only reachable by treating an unstated earnings threshold as a gate. It also conflated having no tax liability with having no refundable credit, which is backwards for a refundable credit."
-us,scenario_066,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model computed the correct answer in its own reasoning (0.0765 × $520 = $39.78) and then discarded it for a fabricated ""IRS EITC table lookup"" value of $234, which corresponds to phase-in on roughly $3,060 of earnings rather than the stated $520. The published EITC table row covering $500–$550 of earned income for a childless filer returns about $40, not $234, so the substituted number contradicts the very table the model cited. Its error is abandoning a correct §32(b) phase-in computation in favor of a hallucinated table entry."
-us,scenario_066,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model asserted that ""EITC requires higher earned income for eligibility,"" imposing a minimum-earnings eligibility floor that §32 does not contain — the childless credit rises linearly at 7.65% from the first dollar earned up to the plateau. With the head aged 40 (inside the 25–64 childless-worker age window) and $520 of wages, the phase-in yields $39.78. Correctly ruling out the CTC for a household with no children does not zero out the EITC."
-us,scenario_066,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"The model flatly stated that the childless EITC is $0 at $520 of earned income, which is only true if the phase-in schedule begins above $520; the actual schedule credits 7.65% of earnings starting at $1. The correct derivation is 0.0765 × $520 = $39.78. The model supplied no threshold or rate, and its answer is consistent with treating the childless EITC as an all-or-nothing credit available only near the plateau."
-us,scenario_066,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model tied EITC availability to having qualifying children and to a supposed income minimum — ""only $520 of earned income and no children"" — but the childless EITC is a separate schedule available to filers aged 25–64, and the head is 40. Phasing in at 7.65% of the stated $520 of wages produces $39.78. Its $0 requires both the nonexistent earnings floor and the incorrect premise that a childless filer gets no EITC."
-us,scenario_066,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model gave no rule or arithmetic, only the assertion that ""at this very low income level, no refundable federal credits are counted,"" which inverts the EITC's design: the credit is largest relative to earnings precisely in the phase-in range that $520 sits in. Applying the 7.65% childless phase-in rate to $520 gives $39.78. Its $0 is consistent with treating refundable credits as an offset to positive tax liability rather than as a payment independent of liability."
-us,scenario_066,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model explicitly invented a statutory floor — ""EITC requires minimum earned income and phase-in; $520 is too low"" — but the phase-in has no lower bound: the childless credit equals 7.65% of earned income from the first dollar until the plateau, so $520 produces $39.78. The model named the correct mechanism (phase-in) and then used it as a disqualifier rather than as the formula. No provision of §32 conditions eligibility on a minimum earnings amount."
-us,scenario_066,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model denied EITC eligibility categorically on the ground that there are no qualifying children, ignoring the childless-worker EITC available to filers aged 25 through 64 — the head is 40 and meets that age test. That pathway phases in at 7.65% of earned income, giving 0.0765 × $520 = $39.78. Ruling out the refundable CTC for a childless household was correct, but the model extended that reasoning to a credit that expressly covers workers without children."
-us,scenario_066,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model stated that ""no qualifying children, so the EITC is not available,"" collapsing the EITC into its with-children schedules and omitting the childless-worker schedule, for which a 40-year-old single filer qualifies under the 25–64 age test. Applying that schedule's 7.65% phase-in rate to the $520 of wages yields $39.78. Its dismissal of the additional child tax credit was correct and does not carry over to the EITC."
-us,scenario_066,head_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model correctly identified that $520 is below Virginia's adult-expansion income limit but then wrongly treated other means-tested health coverage and $14,000 of assets as barriers. Neither current coverage nor assets disqualifies an otherwise eligible MAGI-based expansion adult."
-us,scenario_066,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented a rule that other means-tested health coverage disqualifies an adult from Medicaid. The head instead qualifies through Virginia's ACA adult expansion category because the stated annual MAGI is only 0.03 times FPL.
-us,scenario_066,head_medicaid_eligible,claude-sonnet-5,llm_error,period_annualization,False,"The model replaced the explicitly stated $520 annual wages with $32 × 40 × 52 and falsely asserted that PolicyEngine uses the higher amount. The prompt makes gross wages the annual total and demographic and work facts constant without authorizing wage imputation, so the MAGI test uses the low stated annual income."
-us,scenario_066,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model missed Virginia's ACA adult expansion category, which covers qualifying adults ages 19–64 without requiring disability, parenthood, or another mandatory-category indicator. At age 40 and 0.03 times FPL, the head satisfies that pathway."
-us,scenario_066,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model treated other means-tested health coverage as a Medicaid eligibility exclusion. That coverage fact does not block Virginia ACA adult expansion eligibility, which the head satisfies at 0.03 times FPL."
-us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required a disability or other special categorical indicator and failed to apply Virginia's ACA adult expansion pathway. A 40-year-old adult with MAGI at 0.03 times FPL qualifies without disability or another health-status condition.
-us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model incorrectly made other means-tested health coverage preclusive. It is not an exclusion from the Virginia ACA adult expansion category, under which the head qualifies based on age, status, and MAGI."
-us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit from non-MAGI Medicaid pathways to Virginia's MAGI-based ACA adult expansion category. That category has no asset test, so the $14,000 bank balance does not offset eligibility at 0.03 times FPL."
-us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,period_annualization,False,"The model improperly annualized $32 per hour at 40 hours per week into $66,560 despite the prompt explicitly defining $520 as the full-year gross wage total. It also wrongly treated other means-tested health coverage as disqualifying; using the stated annual income places the head at 0.03 times FPL and within Virginia's adult expansion category."
-us,scenario_066,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"Applied the $2,500/$3,750 SNAP resource limit to the $14,000 bank balance and denied eligibility. It missed that Virginia's TANF non-cash broad-based categorical eligibility confers eligibility, which this household meets. That pathway yields zero net income and the full $298/month maximum ($3,576/year)."
-us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly applied BBCE and found net income of $0, but used about $275/month as the one-person maximum allotment. The FY2026 contiguous-US maximum is $298/month, so the benefit is $3,576/year, not $3,300."
-us,scenario_066,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"Identified the correct $298/month maximum and near-zero net income, then subtracted an invented $1,080/year ($90/month) income offset. The earned income deduction and standard deduction reduce $43.33 of monthly earnings to $0 net income, so the 30% contribution is $0 and the benefit is the full $3,576."
-us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,"Correctly concluded that net income is zero and the maximum allotment applies, but used an outdated maximum of $283/month. The FY2026 one-person maximum is $298/month, or $3,576/year."
-us,scenario_066,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Wrongly asserted that Virginia limits categorical eligibility to TANF/SSI cash recipients, then denied SNAP under a $2,750 asset limit. Virginia's TANF non-cash BBCE makes this household categorically eligible despite its $14,000 in assets, and it receives the $298/month maximum."
-us,scenario_066,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Misread the $520 annual wage total as weekly pay, inflating monthly income to about $2,253 and failing the 130% FPL gross income test. Actual monthly gross income is $43.33, which passes every income test and produces net income of $0 and the $298/month maximum."
-us,scenario_066,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Correctly found $0 net income and the maximum allotment, but used $294/month instead of the FY2026 one-person maximum of $298/month. That gave $3,528 instead of $3,576."
-us,scenario_066,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied the ABAWD three-month time limit on the claim that the head fails the 20-hour work requirement. The head's usual weekly hours are 40, so the work requirement is met and benefits run all 12 months. It also used a $292 maximum instead of $298, giving $876 instead of $3,576."
-us,scenario_066,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Correctly applied BBCE and found net income of $0, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, so 12 months total $3,576, not $3,504."
-us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and found net income of $0, but estimated the one-person maximum at $291/month. The FY2026 maximum is $298/month, or $3,576/year."
-us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Treated very low income as grounds for no benefit, which inverts the SNAP formula; zero net income produces the maximum allotment. With BBCE removing the asset barrier and deductions reducing $43.33 of monthly earnings to $0 net income, the household receives $298/month ($3,576/year), not $0."
-us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly applied BBCE and the maximum-benefit outcome, but used $291/month as the one-person maximum. The FY2026 figure is $298/month, which yields $3,576/year."
-us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly reached the maximum benefit under BBCE, but used a $291/month maximum allotment. The FY2026 one-person contiguous-US maximum is $298/month, or $3,576/year."
-us,scenario_066,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $3,504 answer equals 12 × $292, so it applied a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, and the household's zero net income yields $3,576/year."
-us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and found net income of $0, but its $3,504 annual maximum reflects $292/month. The FY2026 one-person maximum is $298/month, or $3,576/year."
-us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly found net income of $0 and the maximum-allotment outcome, but used $292/month instead of the FY2026 one-person maximum of $298/month."
-us,scenario_066,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Correctly zeroed out net income with the standard deduction, but applied a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, giving $3,576/year."
-us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"Correctly applied BBCE and zeroed net income through the earned income, standard and shelter deductions, but rounded the maximum allotment up to $300/month. The FY2026 one-person maximum is $298/month, so the annual benefit is $3,576, not $3,600."
-us,scenario_066,snap,glm-5.3,llm_error,thresholds_rates,False,"Correctly found net income of $0 under BBCE, but used $297/month as the one-person maximum. The FY2026 maximum is $298/month, which gives $3,576/year instead of $3,564."
-us,scenario_066,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard resource limit. It missed that Virginia's TANF non-cash BBCE makes the household categorically eligible, after which zero net income yields the $298/month maximum ($3,576/year)."
-us,scenario_066,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"Acknowledged the household could qualify but returned $0 with no computation. The correct derivation gives BBCE categorical eligibility and $0 net income after the earned income and standard deductions, so the benefit is the $298/month maximum, or $3,576/year."
-us,scenario_066,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the federal resource limit for nonelderly, nondisabled households to the $14,000 bank balance and denied eligibility. Virginia's TANF non-cash BBCE confers categorical eligibility on this household, and it receives the $298/month maximum."
-us,scenario_066,snap,grok-4.3,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard limit, and cited other health coverage as a factor. Health coverage has no bearing on SNAP, and Virginia's TANF non-cash BBCE makes the household categorically eligible for the $298/month maximum."
-us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"Correctly applied BBCE and zeroed net income with the shelter deduction, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, so 12 months total $3,576."
-us,scenario_066,snap,grok-4.6,llm_error,thresholds_rates,False,"Correctly found net income of $0 under BBCE, but applied a $292/month maximum. The FY2026 one-person maximum is $298/month, which yields $3,576/year instead of $3,504."
-us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and zeroed net income, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, or $3,576/year."
-us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"Correctly concluded that near-zero net income yields the maximum allotment, but rounded it to $300/month. The FY2026 one-person maximum is $298/month, so the annual benefit is $3,576, not $3,600."
-us,scenario_066,snap,kimi-k2.6,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard resource limit and no categorical-eligibility waiver was listed. Virginia's TANF non-cash BBCE applies automatically from the household's income and assets without any listed benefit receipt, and the household receives $298/month."
-us,scenario_066,snap,minimax-m3,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard asset limit. It missed Virginia's TANF non-cash broad-based categorical eligibility, which this household meets and which leads to the full $298/month allotment ($3,576/year)."
-us,scenario_066,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"Stated that net income is far below the poverty line, then still subtracted an $872/year 30% income contribution. The earned income and standard deductions reduce net income to $0, so the contribution is $0. It also used a $292 maximum instead of $298, giving $2,632 instead of $3,576."
-us,scenario_066,snap,qwen3.8-max,llm_error,thresholds_rates,False,"Used a wrong $213/month maximum allotment and subtracted 30% of the $520 annual gross income as if it were a monthly amount, skipping the deductions that reduce net income to $0. It then applied a nonexistent $149/month minimum benefit (the actual minimum is $24). The household receives the full $298/month, or $3,576/year."
-us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,It correctly put the federal childless EITC at about $40 (7.65% phase-in) and knew Virginia pays a refundable EITC. But it used the outdated 15% rate and rounded to $6. The current 20% rate gives 0.20 × $39.78 = $7.96.
-us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed Virginia has no refundable credits and that the EIC is federal-only. That misses Virginia's refundable EITC, which equals 20% of the federal EITC ($39.78 here) and is paid regardless of liability, giving $7.96."
-us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It said a childless filer with $520 of wages gets essentially no federal EITC because of a minimum earned-income threshold. No such minimum exists: the credit phases in at 7.65% from the first dollar, giving $39.78. It also cited a 15% Virginia rate instead of 20%; the correct refundable credit is $7.96."
-us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It identified Virginia's refundable EITC but dismissed it as negligible and rounded it to $0, while also citing the old 15% rate. The credit has no minimum amount and does not depend on liability. It is 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated Virginia's EITC/low-income credit as capped at tax liability, which is $0 here. It ignored the separate refundable EITC option, which pays 20% of the $39.78 federal EITC ($7.96) even with zero liability."
-us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It claimed the federal childless EITC needs earned income above about $600 and is $0 at $520. That is wrong: the credit phases in at 7.65% of earnings from the first dollar, giving $39.78. Virginia's refundable EITC is 20% of that, or $7.96."
-us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It zeroed the credit on the grounds that the refundable EITC option was never elected. The prompt says to assume take-up, so the household claims the refundable EITC: 20% of the $39.78 federal EITC, or $7.96."
-us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It asserted that Virginia's earned income credit is only nonrefundable. That misses the refundable EITC, which pays 20% of the federal EITC; with a $39.78 federal credit, that is $7.96."
-us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed the federal EITC exactly ($39.78) but used the outdated 15% Virginia rate, giving $5.97. The current refundable rate is 20%, which gives $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It said no state refundable credits apply and never recognized Virginia's refundable EITC. That credit is 20% of the $39.78 federal childless EITC, or $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the old 15% Virginia refundable EITC rate to a federal EITC it rounded to $40, getting $6. The correct calculation is 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly derived the $39.78 federal EITC but used the old 15% Virginia refundable rate, giving $5.97. The current rate is 20%, which gives $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It said Virginia has no refundable individual credits for this profile. It missed the refundable EITC, which pays 20% of the $39.78 federal childless EITC ($7.96) to a worker with no children."
-us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed $39.78 × 15% = $5.97 using the superseded 15% Virginia refundable EITC rate. The current rate is 20%, which gives $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly used the $39.78 federal EITC but applied the old 15% Virginia rate, giving $5.97. The current 20% refundable rate gives $7.96."
-us,scenario_066,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,It reasoned that a household with no tax liability has no refundable credits. That inverts how refundability works: Virginia's refundable EITC pays 20% of the $39.78 federal EITC ($7.96) even with zero liability.
-us,scenario_066,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It called the Virginia EITC nonrefundable and therefore useless against $0 tax. That misses the refundable EITC, which pays 20% of the $39.78 federal EITC, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It concluded that no Virginia refundable credit is triggered. That overlooks the refundable EITC, which is triggered by the $39.78 federal childless EITC on $520 of wages and pays 20% of it, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It assumed no state refundable credits at this income. Virginia's refundable EITC applies at exactly this low income: 20% of the $39.78 federal EITC, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly recognized Virginia's refundable EITC and a federal childless EITC of about $40. But it applied the outdated 15% rate to get $6 instead of the current 20% rate, which gives $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It ruled out any Virginia refundable credit because the filer is childless with $520 of wages. Childless workers do get the federal EITC ($39.78 here), and Virginia's refundable EITC pays 20% of it, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated having no children as disqualifying for any Virginia refundable credit. The childless federal EITC is $39.78, and Virginia's refundable EITC pays 20% of it, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It said no refundable Virginia credit applies to a childless filer at this income. It missed that the $39.78 federal childless EITC carries through to Virginia's refundable EITC at 20%, giving $7.96."
-us,scenario_066,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It reasoned that zero tax liability means no refundable credits. Refundable credits are paid regardless of liability, and Virginia's refundable EITC is 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It said no Virginia refundable credit applies to a single adult with $520 of income. It overlooked the refundable EITC, which pays 20% of that worker's $39.78 federal EITC, or $7.96."
-us,scenario_066,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the outdated 15% Virginia refundable EITC rate to a federal EITC rounded to $40, getting $6. The correct calculation is 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It claimed Virginia's 15% refundable EITC election expired before tax year 2026. In fact the refundable EITC is still available in 2026, at 20% of the federal EITC, so it pays 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It asserted that Virginia has no state EITC. Virginia has a refundable EITC equal to 20% of the federal EITC, which here is 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the old 15% Virginia refundable EITC rate on a federal EITC rounded to $40, getting $6. The current 20% rate on the exact $39.78 gives $7.96."
-us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,It tied Virginia refundable credits to having state tax liability and to having dependents. Virginia's refundable EITC pays a childless worker with zero liability 20% of the federal EITC: 20% × $39.78 = $7.96.
-us,scenario_066,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It claimed Virginia's refundable EITC applied only to tax years 2022–2025 at 15%. In fact the refundable EITC still applies in 2026 at 20% of the federal EITC, so it pays 20% × $39.78 = $7.96."
-us,scenario_066,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no Virginia refundable credits apply and never considered the refundable EITC. That credit pays 20% of the $39.78 federal childless EITC, or $7.96."
-us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,It said Virginia's EITC requires tax liability and offers no refundable payment to a childless filer at this income. The refundable EITC option pays 20% of the $39.78 federal EITC ($7.96) whether or not there is liability.
-us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It treated having no qualifying children as disqualifying for Virginia refundable credits. The childless federal EITC is $39.78, and Virginia's refundable EITC pays 20% of it, or $7.96."
+us,scenario_066,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It said the EITC requires substantial earned income and tied refundable credits to having tax liability. In fact, the childless EITC phases in at 7.65% from the first dollar of earnings and is refundable even when no tax is owed. That gives 0.0765 × $520 = $39.78, not $0."
+us,scenario_066,federal_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It reduced earned income by the standard deduction before computing the EITC. The EITC is based on gross earned income (and AGI), not taxable income. The 7.65% phase-in rate applied to the full $520 of wages yields $39.78."
+us,scenario_066,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly computed 7.65% × $520 = $39.78, then dropped that figure for $234. The $234 came from an invented table lookup using an expanded childless credit that does not exist in 2026. Under 2026 law the childless phase-in rate is 7.65%, so no table bracket at $520 comes anywhere near $234."
+us,scenario_066,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It claimed the EITC requires higher earned income to qualify. There is no minimum earnings threshold, and the childless credit phases in at 7.65% from the first dollar. That yields $39.78 on $520 of wages."
+us,scenario_066,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It stated that the EITC for a childless single filer with $520 of earnings is $0. That filer is in the phase-in range, where the credit is 7.65% of earned income: $39.78."
+us,scenario_066,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated having no qualifying children plus very low earnings as ruling out the EITC. A 40-year-old childless worker qualifies for the childless EITC, which phases in at 7.65% of the $520 in wages: $39.78."
+us,scenario_066,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed that very low income produces no refundable credits. It never applied the childless EITC phase-in, which pays 7.65% of earned income from the first dollar and gives $39.78 on $520 of wages."
+us,scenario_066,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It claimed the EITC requires a minimum earned income before the phase-in starts. The phase-in starts at $0 of earnings at 7.65% for childless filers, so $520 of wages produces $39.78."
+us,scenario_066,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It concluded that $520 in wages and no qualifying children make the filer EITC-ineligible. A 40-year-old single filer meets the childless EITC age test, and the credit phases in at 7.65% of earnings, giving $39.78."
+us,scenario_066,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,It treated the lack of qualifying children as making the EITC unavailable and ignored the childless worker EITC for filers aged 25–64. That credit pays 7.65% of the $520 in earnings: $39.78.
+us,scenario_066,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model correctly placed the $520 of income below Virginia's 138% FPL expansion limit, then flipped to ineligible because of the 'other means-tested health coverage' flag and the $14,000 in assets. Neither factor matters for eligibility in the ACA adult expansion category: that category is MAGI-only, has no resource test, and does not depend on current enrollment."
+us,scenario_066,head_medicaid_eligible,claude-haiku-5.5,llm_error,other,False,"The model's reasoning applied the right rule: $520 is below Virginia's 138% FPL expansion threshold, so the head is eligible, and the explanation ends with value = 1. However, it submitted 0, contradicting its own correct conclusion."
+us,scenario_066,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly found $520 of MAGI well under 138% FPL, then wrongly claimed that existing means-tested health coverage disqualifies the head under PolicyEngine rules. Medicaid eligibility in the adult expansion category is based only on age, immigration status and MAGI, and current coverage has no effect on it."
+us,scenario_066,head_medicaid_eligible,claude-sonnet-5,llm_error,period_annualization,False,"The model replaced the stated $520 of annual wages with an imputed $32 × 40 hours × 52 weeks ≈ $66,560 and compared that to 138% FPL. MAGI uses actual annual wages ($520, about 0.03x FPL); the hourly rate and usual hours are not imputed into countable income."
+us,scenario_066,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said the head fits no coverage category, missing that Virginia's Medicaid expansion creates an adult category for non-disabled 19-64-year-olds with MAGI at or below 138% FPL. The head, at age 40 with income of 0.03x FPL, qualifies in that category."
+us,scenario_066,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the 'other means-tested health coverage at interview' flag as disqualifying. Medicaid eligibility in the ACA adult category is determined by age, immigration status and MAGI (0.03x FPL here), not by current enrollment in other coverage."
+us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required a disability or other categorical indicator and overlooked that Virginia's ACA expansion covers childless, non-disabled adults on income alone. With MAGI of $520 (0.03x FPL), the 40-year-old head is eligible in the ADULT category."
+us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model said other means-tested coverage rules out Medicaid. Current coverage does not affect eligibility, and the head qualifies in Virginia's expansion adult category with MAGI of 0.03x FPL."
+us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit to the $14,000 bank balance. MAGI-based categories, including Virginia's ACA adult expansion category, have no resource test, so the head is eligible on $520 of income alone."
+us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,period_annualization,False,"The model annualized the $32 hourly wage at 40 hours a week into about $66,560 of earnings instead of using the stated $520 of annual wages. It also treated existing means-tested coverage as disqualifying. With actual MAGI of 0.03x FPL, the head meets the expansion adult limit, and current coverage has no bearing on eligibility."
+us,scenario_066,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied the federal $2,500/$3,750 SNAP resource limit to the $14,000 bank balance. It missed that Virginia's broad-based categorical eligibility makes the household categorically eligible through the TANF non-cash tests, so the federal asset limit does not apply. The household gets the full $298/month maximum, or $3,576 a year."
+us,scenario_066,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It ruled the household ineligible because $14,000 exceeds the federal SNAP resource limit. It ignored Virginia's broad-based categorical eligibility, under which the household passes the TANF non-cash asset test and the federal resource limit is waived. The correct benefit is the $298/month one-person maximum."
+us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly applied BBCE and found net income of $0, but it used a one-person maximum allotment of $275/month. The FY2026 contiguous-US maximum is $298/month, which gives $3,576 a year, not $3,300."
+us,scenario_066,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It correctly found net income of zero and the $298 maximum ($3,576 a year). It then subtracted an invented 'small income offset' to get $2,496 (about $208/month). With $0 net income, the 30% net-income contribution is $0, so the benefit equals the full maximum allotment."
+us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly found net income near zero and the household entitled to the maximum allotment, but it used $283/month instead of the FY2026 one-person maximum of $298/month. That understates the annual benefit by $180."
+us,scenario_066,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It wrongly asserted that Virginia limits categorical eligibility to TANF/SSI recipients and then applied the $2,750 federal asset limit to the $14,000 balance. Virginia runs broad-based categorical eligibility through TANF non-cash benefits, which this household's income and assets satisfy. That waives the federal resource test and yields the $298/month maximum."
+us,scenario_066,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It misread the $520 annual wage total as weekly earnings and computed about $2,253/month, so the household failed the 130% FPL gross income test. Actual monthly earnings are $43.33, well under every income limit, and the household gets the full $298/month maximum."
+us,scenario_066,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly found net income of $0, but it used a $294/month one-person maximum allotment. The FY2026 contiguous-US maximum is $298/month, so the annual benefit is $3,576, not $3,528."
+us,scenario_066,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It cut benefits to 3 months under the ABAWD time limit even though the head usually works 40 hours a week, which satisfies the 20-hour work requirement. The household therefore passes the work rules in all 12 months. It also used the outdated $292 maximum instead of $298."
+us,scenario_066,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly applied Virginia BBCE and found net income of $0, but it used the FY2025 one-person maximum of $292/month. The FY2026 maximum is $298/month, so the correct total is $3,576, not $3,504."
+us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly waived the asset test under BBCE and found net income of $0, but it used a $291/month maximum allotment. The FY2026 one-person contiguous-US maximum is $298/month, or $3,576 a year."
+us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It returned $0 without applying any actual rule. It never recognized that Virginia's broad-based categorical eligibility waives the asset test, or that $43.33 of monthly earnings leaves net income at $0. Correctly applied, those rules give the full $298/month one-person maximum."
+us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly applied BBCE and the maximum-benefit logic, but it used $291/month instead of the FY2026 one-person maximum of $298/month. That understates the annual benefit by $84."
+us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly applied Virginia BBCE and found net income of zero, but it used a $291/month maximum allotment. The FY2026 one-person maximum is $298/month, giving $3,576 a year."
+us,scenario_066,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $3,504 equals 12 × $292, the FY2025 one-person maximum allotment. The FY2026 maximum is $298/month, so the full-benefit household receives $3,576."
+us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly waived the asset test under BBCE and found net income of $0, but its $3,504 annual maximum reflects the FY2025 figure of $292/month instead of the FY2026 figure of $298/month."
+us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used $292/month, the FY2025 one-person maximum allotment. The FY2026 maximum is $298/month, so the annual benefit is $3,576."
+us,scenario_066,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly found net income of $0, but it applied the FY2025 one-person maximum of $292/month instead of the FY2026 maximum of $298/month. That understates the benefit by $72."
+us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"Its deductions and BBCE treatment were correct, but it rounded the one-person maximum allotment to about $300/month. The FY2026 contiguous-US maximum is exactly $298/month, or $3,576 a year."
+us,scenario_066,snap,glm-5.3,llm_error,thresholds_rates,False,"It correctly applied BBCE and found net income of $0, but it used $297/month as the one-person maximum. The FY2026 maximum is $298/month, so the annual total is $3,576, not $3,564."
+us,scenario_066,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It disqualified the household for having $14,000 in bank assets under 'typical resource limits'. It missed that Virginia's broad-based categorical eligibility removes the federal SNAP asset test for this household. The household is eligible for the $298/month maximum."
+us,scenario_066,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its own reasoning acknowledged that SNAP is available at this very low income, yet it returned $0 without citing any disqualifying rule. With $43.33 of monthly earnings, Virginia BBCE and $0 net income, the household gets the full $298/month maximum."
+us,scenario_066,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied the federal resource limit for a nonelderly, nondisabled household to the $14,000 balance. It overlooked Virginia's broad-based categorical eligibility, which waives that limit because the household meets the TANF non-cash tests."
+us,scenario_066,snap,grok-4.3,llm_error,thresholds_rates,False,"It denied SNAP because the $14,000 in assets exceeds the federal limit, and it also cited the household's other health coverage, which has no bearing on SNAP. Virginia's broad-based categorical eligibility waives the asset test, so the household receives the $298/month maximum."
+us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly applied BBCE and the shelter deduction to reach $0 net income, but it used an estimated $292/month maximum (the FY2025 figure). The FY2026 one-person maximum is $298/month."
+us,scenario_066,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly found BBCE eligibility and $0 net income, but it used the outdated $292/month one-person maximum allotment instead of the FY2026 figure of $298/month. That gives $3,504 instead of $3,576."
+us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It reached net income of zero and the maximum benefit (even though it misstated the standard deduction as $204 a year), but it applied the FY2025 maximum allotment of $292/month. The FY2026 one-person maximum is $298/month."
+us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"It correctly treated the household as getting the maximum allotment, but it rounded that allotment to $300/month. The FY2026 one-person contiguous-US maximum is $298/month, which gives $3,576 a year."
+us,scenario_066,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It applied the federal resource limit because 'no categorical-eligibility waiver is listed'. Virginia's broad-based categorical eligibility applies automatically through TANF non-cash benefits, and this household passes those tests, so the $14,000 in assets does not disqualify it."
+us,scenario_066,snap,minimax-m3,llm_error,thresholds_rates,False,"It recognized the income was far below the limits, but it disqualified the household on the federal asset limit. It missed that Virginia's broad-based categorical eligibility waives that limit, leaving the household entitled to the $298/month maximum."
+us,scenario_066,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It subtracted an $872 '30% income contribution', which is not 30% of anything in this case: 30% of the $520 gross is only $156, and the contribution should be 30% of net income, which is $0 after the standard and shelter deductions. It also used the outdated $292 maximum instead of $298/month."
+us,scenario_066,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It used a fabricated $213/month maximum allotment, subtracted 30% of annual gross income from a monthly figure, and invented a $149/month minimum benefit floor. The correct calculation gives net income of $0 after deductions and the FY2026 one-person maximum of $298/month."
+us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly found the refundable Virginia EITC and a federal childless EITC of about $40, but used the old 15% rate. Since tax year 2024, Virginia's refundable EITC has been 20% of the federal credit, so the answer is 20% × $39.78 = $7.96, not about $6."
+us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said Virginia has no refundable individual credits and treated the EITC as federal-only. Virginia does have a refundable EITC equal to 20% of the federal EITC, which here is 20% × $39.78 = $7.96. The model also described $520 as $32 × 40 hours, but $520 is simply the listed annual wage."
+us,scenario_066,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Said no refundable Virginia credit applies and missed the refundable EITC. That credit is 20% of the federal EITC. The federal EITC is 7.65% × $520 = $39.78, so the state credit is $7.96."
+us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Treated the federal childless EITC as effectively zero at $520 of earnings. The credit starts at the first dollar earned, at 7.65%, so it is $39.78. The model also used 15% instead of Virginia's 20% refundable rate, which gives $7.96."
+us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Found Virginia's refundable EITC but rounded it to zero, assuming it depends on tax liability. The refundable EITC is paid even when tax is zero. It equals 20% of the $39.78 federal EITC, or $7.96. The model also used the outdated 15% rate."
+us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Applied only the nonrefundable Virginia low-income credit/EITC option, limited by zero tax. It missed that Virginia also offers a refundable EITC at 20% of the federal EITC, which pays $7.96 here even with no tax liability."
+us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Claimed the federal childless EITC needs earnings above about $600. The phase-in starts at $1 at 7.65%, so $520 of wages produces a $39.78 federal EITC. Virginia's 20% refundable EITC on that is $7.96, not $0."
+us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Assumed the refundable Virginia EITC applies only if elected and that no election was made. The prompt says to assume take-up of credits when required. The refundable EITC is 20% of the $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Said Virginia's earned income credit is nonrefundable only. Virginia also has a refundable EITC equal to 20% of the federal EITC, which here is 20% × $39.78 = $7.96."
+us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Got the federal EITC exactly right at $39.78 but applied the pre-2024 15% rate. Virginia's refundable EITC is 20% of the federal credit, so the answer is $7.96, not $5.97."
+us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Said no state refundable credits apply and missed Virginia's refundable EITC. That credit is 20% of the federal EITC of 7.65% × $520 = $39.78, which gives $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Found the refundable EITC but used 15% and a rounded $40 federal EITC. Virginia's rate is 20%, applied to the exact $39.78 federal EITC, which gives $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Got the federal EITC right at $39.78 but applied the outdated 15% rate. Since tax year 2024, Virginia's refundable EITC is 20% of the federal credit, which is $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Said Virginia has no refundable credits for this profile and missed the refundable EITC. That credit is 20% of the $39.78 federal childless EITC, or $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,Correctly used the $39.78 federal EITC but applied 15% instead of Virginia's current 20% refundable EITC rate. The answer is $7.96.
+us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly used the $39.78 federal EITC but applied the old 15% rate. Virginia's refundable EITC is 20% of the federal credit, which is $7.96."
+us,scenario_066,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"Concluded that zero tax liability rules out refundable credits. Virginia's refundable EITC is paid regardless of liability, at 20% of the $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"Said the Virginia EITC is nonrefundable. Virginia offers a refundable EITC at 20% of the federal EITC, which pays $7.96 here despite $0 state tax."
+us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Said no refundable Virginia credit is triggered. The $520 of earnings produces a $39.78 federal EITC, which triggers Virginia's 20% refundable EITC of $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Assumed no state refundable credits at this income level. Virginia's refundable EITC is 20% of the $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Found the refundable Virginia EITC but used 15% of about $40. The current rate is 20% of the exact $39.78 federal EITC, which gives $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Concluded that a childless single adult gets no refundable Virginia credit. Childless filers do get a federal EITC (7.65% × $520 = $39.78), and Virginia refunds 20% of it, or $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,Assumed a childless household cannot get a Virginia refundable credit. The childless federal EITC of $39.78 qualifies the head for Virginia's 20% refundable EITC of $7.96.
+us,scenario_066,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"Said no refundable Virginia credit applies to a childless filer at these earnings. The $39.78 federal childless EITC passes through to Virginia's 20% refundable EITC, which is $7.96."
+us,scenario_066,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Assumed refundable credits need tax liability. A refundable credit is paid even with zero liability, and Virginia's refundable EITC here is 20% of $39.78, or $7.96."
+us,scenario_066,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"Missed Virginia's refundable EITC entirely. That credit is 20% of the federal EITC, which is 7.65% × $520 = $39.78, so the state credit is $7.96."
+us,scenario_066,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied the outdated 15% rate to a rounded $40 federal EITC. Virginia's refundable EITC is 20% of the exact $39.78 federal EITC, which is $7.96."
+us,scenario_066,state_refundable_credits,grok-4.7,llm_error,categorical_eligibility,False,"Claimed Virginia's refundable EITC expired before 2026. The refundable EITC remains in effect at 20% of the federal EITC, which gives $7.96 for this household."
+us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Said Virginia has no state EITC. Virginia has a refundable EITC equal to 20% of the federal EITC, which here is 20% × $39.78 = $7.96."
+us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used 15% of a rounded $40 federal EITC. Virginia's refundable EITC rate is 20% applied to $39.78, which gives $7.96."
+us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"Treated zero state tax and no dependents as ruling out any refundable credit. Virginia's refundable EITC is paid regardless of liability and covers childless filers, giving 20% × $39.78 = $7.96."
+us,scenario_066,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"Claimed Virginia's refundable EITC applied only for 2022–2025. The refundable EITC is still in effect for 2026, at 20% rather than the 15% the model cited, so the answer is 20% × $39.78 = $7.96."
+us,scenario_066,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Missed Virginia's refundable EITC. That credit is 20% of the $39.78 federal childless EITC, which is $7.96."
+us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Assumed Virginia's EITC needs tax liability and does not pay out beyond it. Virginia's refundable EITC pays 20% of the federal EITC in full, so the household receives $7.96 with $0 tax."
+us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Concluded that a filer with no qualifying children gets no refundable Virginia credit. Childless filers get the $39.78 federal EITC, and Virginia refunds 20% of it, or $7.96."
us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1's ACA expansion eligibility on the parents' tax-unit MAGI (about $87,300 for 3 people) and also applied parental income and resources to the disability pathway. PolicyEngine puts dependent1 in the ADULT expansion category at 0.00×FPL, which is far under 138%, so dependent1 is eligible."
us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It tested dependent1 against the 138% FPL adult limit using the whole tax unit's income. PolicyEngine measures dependent1's MAGI income level at 0.00×FPL in the ACA adult expansion category, so the parents' income does not disqualify dependent1."
us,scenario_067,dependent1_medicaid_eligible,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the whole filing unit as dependent1's MAGI household and counted $98,102, which combines parents' wages, pension and dependent1's disability benefits, against 138% FPL. PolicyEngine puts dependent1's MAGI income level at 0.00×FPL: the parents' income is not attributed and the disability benefits are not MAGI income. Dependent1 qualifies under the ACA adult expansion."
@@ -5464,61 +5563,111 @@ us,scenario_067,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household
us,scenario_067,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It chose the right program, Indiana HIP for adults up to 138% FPL, but tested dependent1 against the household's roughly $87K MAGI. PolicyEngine puts dependent1's own MAGI income level at 0.00×FPL, so dependent1 qualifies."
us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It wrongly treated Indiana as limited on expansion-type pathways and tested dependent1 against 3-person household MAGI of about $87,302. Indiana has adopted the ACA expansion, and PolicyEngine puts dependent1 in the ADULT category at 0.00×FPL, so dependent1 is eligible."
us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed dependent1's $10,800 in disability benefits exceed Indiana's adult Medicaid limit. The benefits are not MAGI income, so PolicyEngine puts dependent1 at 0.00×FPL, and $10,800 alone is still under 138% FPL for a single adult."
-us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Treated the disabled flag plus generic disability benefits as satisfying a disability-based Medicare rule. Medicare's under-65 disability pathway requires 24 months of Social Security disability entitlement, and this household records neither SSDI entitlement nor any months of receipt."
-us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Relabeled the generic $10,800 'disability benefits' as SSDI and then declared the 24-month waiting period satisfied because the disabled status is constant for the year. A status that holds all year proves no prior 24 months of SSDI entitlement, and the prompt makes unlisted SSDI receipt and waiting-period months false, so the Medicare disability pathway fails."
-us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"Assumed the disability benefits were SSDI and granted Medicare despite naming the 24-month waiting period itself. The household lists no SSDI entitlement and no months of receipt, so under the prompt's unlisted-is-false rule neither the SSDI condition nor the waiting period is met, and a 23-year-old is not Medicare-eligible."
-us,scenario_067,dependent1_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Treated disability status alone as conferring Medicare eligibility. Under-65 Medicare requires 24 months of Social Security disability entitlement or ESRD, and neither appears for this 23-year-old."
-us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"Read generic annual disability benefits as support for under-65 Medicare eligibility. It never checked the actual requirement of 24 months of Social Security disability entitlement, which the household does not record."
-us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"Applied disability-based Medicare because the dependent is disabled and receives disability benefits. That pathway requires 24 months of SSDI entitlement, and the listed disability benefits are not SSDI entitlement with a completed waiting period."
-us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Invoked a 'modeled disability-based Medicare pathway' triggered by reported disability plus disability benefits. That pathway needs 24 months of Social Security disability entitlement, and this household records none, so the dependent does not qualify."
-us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"Claimed that PolicyEngine treats a disabled person receiving disability benefits as Medicare-eligible under 65. Eligibility under 65 requires 24 months of Social Security disability entitlement or ESRD, and neither is present here."
-us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"Concluded Medicare eligibility 'under disability rules' from the disabled flag and generic disability benefits. It skipped the required 24 months of Social Security disability entitlement, which the household does not have."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning reached $5,347 after using a $32,600 standard deduction (the 2026 figure is $32,200) and an auto-loan interest deduction that the facts do not support. It then submitted $3,391, a number no step of its own derivation produces."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Everything else matches the reference: $32,200 standard deduction, 10%/12% brackets at $24,800, and the $500 dependent credit. But it also subtracted the $1,845 OBBBA auto-loan interest deduction, though no qualified new, US-assembled vehicle loan is established. That cut taxable income to $53,257 and understated tax by exactly $221.40 (12% × $1,845)."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction instead of the 2026 $32,200, which gave taxable income of $58,802 instead of $55,102. It then subtracted an invented ~$1,266 of nonrefundable credits instead of the single $500 credit for other dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its first pass, with a ~$32,300 deduction and a 10% bracket to ~$24,000, landed near the reference at ~$5,620. It then switched to 2025 bracket edges ($23,850) and submitted $6,280 with no arithmetic supporting that figure, instead of $55,102 taxed at the 2026 brackets less the $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized using ~$6,500 of imputed mortgage interest (unlisted, so $0) and claimed its ~$23,121 itemized total beat a ~$32,300 standard deduction, which is false by its own numbers. It then replaced its computed $6,722 with an unsupported $5,594 instead of using the $32,200 standard deduction and taxable income of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the reference path exactly: taxable income $55,102, tax ~$6,116, less the $500 other-dependent credit = ~$5,616. It then discarded that result for an unexplained 'bracket estimate' of $6,104."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It applied the correct $32,200 standard deduction, 2026 brackets, and $500 other-dependent credit, but also deducted the $1,845 auto-loan interest with no qualified vehicle loan established. That lowered taxable income to $53,257 and tax by $221.40."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 figure to a $30,000 standard deduction instead of the enacted 2026 $32,200 and used a guessed $24,500 10% bracket edge instead of $24,800. That overstated taxable income at $57,302 and gave $5,886 after the $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"With a $32,600 standard deduction (the 2026 amount is $32,200) it derived $6,068 less the $500 other-dependent credit. It then submitted $10,500, citing 'self-employment-like adjustments' and 'benefit phase-outs' that do not apply to this all-wage household."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put the dependent's $10,800 disability benefits into the joint AGI ($98,102 instead of $87,302) and used a $30,600 standard deduction instead of $32,200. It also denied the $500 credit for other dependents because the dependent is 23, but a disabled dependent of any age qualifies for that credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used a $16,500 standard deduction, three $5,250 personal exemptions, and a 15% bracket. OBBBA made the $32,200 standard deduction, zero exemptions, and the 10%/12% brackets permanent. It also omitted the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: itemized medical plus Indiana tax, three $5,300 personal exemptions, and a 15% bracket. The correct rules are the permanent $32,200 standard deduction, no exemptions, and 12% above $24,800. It also dropped the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,486.84 matches this computation: $87,302 − $31,500 (the 2025 standard deduction) − $1,845 auto-loan interest = $53,957, taxed with a $24,400 10% bracket, less $500. The auto-loan deduction does not apply without a qualified vehicle loan, and the 2026 parameters are a $32,200 deduction and a $24,800 bracket edge."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It modeled a TCJA sunset: $15,150 of personal exemptions, a $15,900 standard deduction, and a 15% bracket. It also added 85% of the dependent's disability benefits ($9,180) to the parents' AGI and omitted the $500 other-dependent credit. The correct result uses AGI of $87,302 less the permanent $32,200 standard deduction."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reported taxable income of ~$60,400, though $87,302 minus its own $31,200 deduction is $56,102 and the correct figure is $55,102. It also treated the household as having no qualifying dependents, dropping the $500 credit for the disabled 23-year-old."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, subtracting $16,050 of personal exemptions plus itemized deductions and taxing in a 15% bracket. The correct rules are the permanent $32,200 standard deduction and 10%/12% brackets. It also omitted the $500 credit for other dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules with $15,150 of personal exemptions and a 15% bracket. It itemized with $4,500 of mortgage interest the facts do not list (unlisted inputs are $0). The correct approach uses the $32,200 standard deduction and subtracts the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $7,534 is higher than even the reference's pre-credit tax of $6,116.24. It matches taxable income of about $66,900: the dependent's $10,800 benefits added to AGI ($98,102), minus a ~$31,200 standard deduction, with no $500 other-dependent credit. The correct base is $87,302 − $32,200 = $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 $32,200 and the 2025 $23,850 10% bracket edge instead of $24,800, overstating taxable income at $56,502. The $500 credit was applied correctly."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,912.24 matches taxable income of $57,302, from a $30,000 standard deduction instead of $32,200, taxed with a $23,200 10% bracket edge instead of $24,800, before the correct $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction instead of the 2026 $32,200 and the 2025 $23,850 10% bracket edge instead of $24,800. That gave taxable income of $57,302 and $5,899.24 after the correct $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $30,000 standard deduction instead of the enacted 2026 $32,200 and used a $23,600 10% bracket edge instead of $24,800. That overstated taxable income at $57,302, though it correctly subtracted the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied any dependent credit by applying the qualifying-relative gross income test to the dependent's $10,800 of benefits. A permanently and totally disabled child has no age limit as a qualifying child and faces no gross income test, so the $500 other-dependent credit applies. It also used a $25,200 10% bracket edge instead of $24,800."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $4,567 is $1,049 below the reference. That equals removing about $8,700 of taxable income beyond the $32,200 standard deduction, and no deduction or credit in the facts supports it. The correct figure is $55,102 taxed at 10%/12% ($6,116.24) less only the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It stacked the standard deduction with itemized mortgage and medical deductions. Mortgage interest is unlisted (so $0), and the $16,621 of medical deductions is well below the $32,200 standard deduction. That understated taxable income by roughly $5,600 and gave $4,940."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated 85% of the dependent's $10,800 disability benefits ($9,180) as taxable Social Security on the parents' joint return. Those benefits belong to the dependent and stay out of the joint AGI. This raised taxable income to $64,282 and overstated tax by exactly $1,101.60 (12% × $9,180)."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its round $8,500 exceeds even the pre-credit tax of $6,116.24. It matches taxable income about $20,000 above the correct $55,102 ($87,302 − $32,200) with no $500 other-dependent credit subtracted."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules with $15,750 of personal exemptions, itemized medical plus SALT, and higher brackets, and it allowed no nonrefundable credits. Under permanent OBBBA law, the $32,200 standard deduction and 10%/12% brackets apply, less the $500 credit for the disabled dependent."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using a $16,600 standard deduction, three $5,300 personal exemptions, and a 15% bracket. The permanent 2026 rules are a $32,200 standard deduction, no exemptions, and 12% above $24,800. It also omitted the $500 other-dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law: $15,900 of personal exemptions, itemized deductions, and a 15% bracket. It also declared the other-dependent credit expired, but OBBBA made the $32,200 standard deduction, 10%/12% brackets, and $500 other-dependent credit permanent."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an inflated pre-TCJA $17,291 standard deduction and a 15% bracket, and it skipped the personal exemptions its own sunset framework requires. That gave taxable income of $70,011. The correct taxable income is $55,102, after the permanent $32,200 standard deduction, less the $500 dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents and omitted the $500 credit for other dependents for the disabled 23-year-old, who is a qualifying child regardless of age. It also used the 2025 ~$31,500 standard deduction instead of 2026's $32,200."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 $32,200 and a $24,400 10% bracket edge instead of $24,800, overstating taxable income at $56,502. The $500 other-dependent credit was applied correctly."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It added $9,180 of the dependent's disability benefits to the joint AGI as taxable Social Security, though the benefits belong to the dependent. It also subtracted the $1,845 auto-loan interest deduction with no qualified vehicle loan established. That produced taxable income of $62,437 instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed a $2,000 nonrefundable CTC for the 23-year-old disabled dependent, but the CTC requires a child under 17, so only the $500 credit for other dependents applies. It also added the dependent's $10,800 benefits to AGI and used a $30,000 standard deduction and $23,850 bracket edge instead of $32,200 and $24,800."
-us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the pre-credit tax exactly at $6,116.24 on $55,102. It then applied no nonrefundable credit because the dependent is too old for the CTC, missing that the disabled 23-year-old is a qualifying child for the $500 credit for other dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the dependent's $10,800 disability benefits as taxable income on the parents' return (AGI $98,102 instead of $87,302). It also used a projected $31,200 standard deduction and a $24,150 bracket edge instead of $32,200 and $24,800, overstating taxable income at $66,902."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a fabricated $10,000 nonrefundable CTC that wiped out the tax. The household has no child under 17, so the only nonrefundable credit is the $500 credit for the disabled adult dependent. Its taxable income of $58,150 also exceeds the correct $55,102."
+us,scenario_067,dependent1_medicare_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated being disabled plus receiving disability benefits as enough for Medicare. The under-65 pathway actually requires 24 months of SSDI entitlement (or ESRD/ALS). The listed disability benefits are generic, not SSDI, and months of SSDI receipt are 0, so the 23-year-old does not qualify."
+us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model claimed a disability-based Medicare rule is triggered by disabled status plus disability benefits. Medicare eligibility under 65 requires 24 months of Social Security disability entitlement. The generic disability-benefits income here is not SSDI, and SSDI months are 0, so Dependent 1 is not eligible."
+us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly named the 24-month SSDI waiting period, then decided the $10,800 was SSDI and assumed the waiting period was met. Both steps go against the instructions not to infer unlisted benefit receipt and to treat unlisted numeric inputs (months of SSDI receipt) as 0. With no SSDI and 0 qualifying months, Dependent 1 is not Medicare-eligible."
+us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model relabeled the generic disability benefits as SSDI and treated the 24-month waiting period as satisfied. Neither fact is listed, and SSDI months default to 0. Without 24 months of SSDI entitlement or ESRD, a 23-year-old does not qualify for Medicare."
+us,scenario_067,dependent1_medicare_eligible,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model went straight from ""disabled"" to ""Medicare eligible."" It skipped the requirement that people under 65 need 24 months of SSDI entitlement (or ESRD/ALS). Dependent 1 has 0 SSDI months and only generic disability income, so the person is not eligible."
+us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated annual disability benefits as support for Medicare eligibility under 65. The disability pathway requires 24 months of Social Security disability entitlement specifically, not generic disability income. SSDI months are 0 here, so the answer is not eligible."
+us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model assumed disabled status plus disability benefits triggers disability-based Medicare. That pathway requires 24 months of SSDI entitlement. The listed benefits are a generic disability-income input, not SSDI, and SSDI months are 0, so Dependent 1 is not eligible."
+us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,categorical_eligibility,False,"The model applied a disability-based Medicare pathway based only on reported disability and disability income. That pathway requires 24 months of SSDI receipt, which is 0 for this household, and the generic disability benefits do not count. The 23-year-old dependent is not Medicare-eligible."
+us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model claimed that disability plus disability benefits qualifies a person under 65 for Medicare. Medicare actually requires 24 months of SSDI entitlement or ESRD/ALS. Dependent 1's disability benefits are not SSDI and SSDI months are 0, so the person is not eligible."
+us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied ""disability rules"" without the actual condition, which is 24 months of Social Security disability entitlement before age 65. The generic disability benefits and the disabled flag do not meet that test, and SSDI months are 0. Dependent 1 is not Medicare-eligible."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model took a $1,845 OBBBA auto-loan-interest deduction that isn't available here and used a $32,600 standard deduction instead of the $32,200 2026 MFJ amount. Its own work came to $5,347, but it submitted $3,391, a number that no step of its reasoning supports."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Everything matched the reference except one step: it subtracted the $1,845 household auto loan interest as an OBBBA qualified vehicle loan interest deduction. That deduction requires facts the prompt never gives (a new, U.S.-assembled vehicle financed after 2024), so taxable income stays at $55,102. The extra deduction is exactly why it came in $221.40 low (12% × $1,845)."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction instead of OBBBA's $32,200 for 2026 and estimated about $7,080 of pre-credit tax instead of $6,116.24. It then subtracted $1,266 of credits it never identified, when the only nonrefundable credit is the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Its rough $6,900 estimate overstates the tax on $55,102 of taxable income, which is $6,116.24 using the $32,200 standard deduction and the 10%/12% brackets with a $24,800 breakpoint. It also explicitly assumed no nonrefundable credits, so it never subtracted the $500 credit for the disabled adult dependent."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It reached about $5,620 after the $500 other-dependent credit, then 'adjusted' to $6,280 using a $32,300 standard deduction and 2025 brackets. Those inputs actually give about $5,623, so $6,280 isn't supported by anything in its reasoning. The correct inputs are the $32,200 standard deduction and a $24,800 10% bracket, which give $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It made up about $6,500 of mortgage interest from the $100,000 loan balance, even though unlisted interest counts as $0, and then itemized instead of taking the larger $32,200 standard deduction. Its own computation reached $6,722, but it submitted $5,594, which none of its steps support."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It got the correct path: $55,102 of taxable income, $6,116 of tax, and $5,616 after the $500 other-dependent credit. It then dropped that result for an unexplained 'bracket estimate' of $6,104, which amounts to not taking the $500 credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"On top of the $32,200 standard deduction, it subtracted the $1,845 auto loan interest as a qualified vehicle loan interest deduction. That deduction isn't available because the prompt doesn't establish the new, U.S.-assembled, post-2024-loan conditions. As a result, taxable income was $221.40 of tax too low (12% × $1,845)."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflation-adjusted pre-OBBBA 2025 figures instead of using OBBBA's 2026 parameters. That gave a $30,000 MFJ standard deduction instead of $32,200, which put $2,200 too much in the 12% bracket, and a $24,500 10% bracket instead of $24,800. Those changes produce its $5,886 result."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $6,068 of tax using a $32,600 standard deduction (the correct figure is $32,200) and acknowledged the $500 other-dependent credit. It then submitted $10,500, citing 'self-employment-like adjustments' and 'benefit phase-outs' that have no place in a wage-and-pension return."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of nontaxable disability benefits to the couple's AGI and used a $30,600 standard deduction instead of $32,200. It also said no nonrefundable credit applies because the dependent is over 17. That misses the $500 credit for other dependents: as a permanently disabled qualifying child, the dependent is eligible regardless of age."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the expired pre-TCJA regime ($16,500 standard deduction, three $5,250 personal exemptions, 15% bracket). OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no exemptions, and a 12% second bracket. It also left out the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset: it itemized medical expenses plus state tax, took three $5,300 personal exemptions, and applied a 15% bracket. Under OBBBA, 2026 uses the $32,200 standard deduction (larger than its $19,108 of itemized deductions), no exemptions, and 10%/12% brackets. It also never subtracted the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted an auto-loan-interest deduction that doesn't apply, because the qualified-vehicle conditions aren't established, which pushed taxable income below the correct $55,102. Without that deduction the tax is $6,116.24, and after the $500 other-dependent credit it is $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the pre-TCJA regime (personal exemptions and a 15% bracket) instead of OBBBA's 2026 $32,200 standard deduction with 10%/12% brackets. It also treated the dependent's disability benefits as Social Security and added $9,180 of 'taxable' benefits to AGI. Finally, it didn't apply the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $31,200 standard deduction instead of $32,200, and its stated $60,400 of taxable income doesn't even match $87,302 minus that deduction. It also said there were no significant nonrefundable credits, which leaves out the $500 credit for the disabled adult dependent."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It wrongly treated 2026 as a TCJA-expiration year: it itemized $19,064, subtracted $16,050 of personal exemptions, and taxed income at 15%. OBBBA keeps the $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets, and it also allows the $500 other-dependent credit, which the model didn't apply."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA rules (personal exemptions and a 15% bracket) instead of OBBBA's 2026 $32,200 standard deduction and 10%/12% brackets. It also invented $4,500 of mortgage interest, although only the loan balance was given and unlisted interest counts as $0."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The correct calculation is $87,302 − $32,200 = $55,102 of taxable income, $6,116.24 of tax at 10%/12% with the $24,800 breakpoint, and $5,616.24 after the $500 other-dependent credit. Its $7,534 overstates taxable income and does not subtract that credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of OBBBA's $32,200 for 2026 MFJ, and it ended the 10% bracket at $23,850 (the 2025 figure) instead of $24,800. Together these overstated tax before the correctly applied $500 credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The rest of its structure was right (MFJ AGI and the $500 other-dependent credit). Its $5,912.24 matches stale parameters, roughly a $30,000 standard deduction and an earlier-year 10% bracket. With 2026's $32,200 standard deduction and $24,800 10% bracket, the result is $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used pre-OBBBA 2025 figures: a $30,000 standard deduction and a $23,850 10% bracket. The 2026 MFJ figures are $32,200 and $24,800, so its taxable income was $2,200 too high and too little income fell in the 10% bracket."
+us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and a $23,600 '2025 proxy' for the end of the 10% bracket. OBBBA's 2026 MFJ figures are a $32,200 standard deduction and a $24,800 10% bracket, so its taxable income and tax were both too high."
+us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied the $500 credit for other dependents by applying the qualifying-relative gross-income test. A permanently and totally disabled child of any age is a qualifying child, so that test doesn't apply, and the dependent qualifies for the credit. It also ended the 10% bracket at $25,200 instead of $24,800."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $4,567 understates the liability. With the $32,200 standard deduction, taxable income is $55,102 and tax is $6,116.24 at 10%/12%. The only nonrefundable credit is the $500 other-dependent credit, which leaves $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It brought in itemized mortgage and medical deductions, but no mortgage interest amount was given and medical expenses over the 7.5% floor ($16,621) fall short of the $32,200 standard deduction. Its $4,940 is below the correct $6,116.24 of tax minus the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 of disability benefits as Social Security and added 85% ($9,180) to the joint return's income. Those benefits are the dependent's own nontaxable disability income and stay out of the couple's AGI, so taxable income is $55,102, not $64,282."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $8,500 overstates the tax. Wages plus pension less the $32,200 standard deduction gives $55,102 of taxable income, which produces $6,116.24 at the 2026 10%/12% rates, and the $500 other-dependent credit brings it to $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA' rules: itemized deductions of medical plus SALT, $15,750 of personal exemptions, and a 15% bracket. OBBBA made the TCJA permanent, so 2026 has a $32,200 standard deduction, no exemptions, and 10%/12% brackets. It also missed the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset and used a $16,600 standard deduction, three $5,300 exemptions, and a 15% bracket. Under OBBBA, 2026 has a $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets, and the $500 other-dependent credit applies, which the model also left out."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the expired pre-TCJA structure (itemizing, $15,900 of exemptions, 15% bracket) and wrongly said the other-dependent credit had expired. OBBBA made the $500 credit permanent and set the 2026 MFJ standard deduction at $32,200 with 10%/12% brackets."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It indexed a pre-TCJA $17,291 standard deduction and applied a 15% bracket, and it left out personal exemptions even under that framework. The 2026 rules under OBBBA are a $32,200 standard deduction and 10%/12% brackets, with a $500 other-dependent credit that it also didn't apply."
+us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents, so it left out the $500 credit for other dependents. The permanently disabled 23-year-old is a qualifying child, because disability waives the age test, and is eligible for that credit. It also used a $31,500 standard deduction instead of the 2026 figure of $32,200."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of OBBBA's $32,200 for 2026 MFJ and ended the 10% bracket at $24,400 instead of $24,800, which overstated taxable income and tax. It applied the $500 other-dependent credit correctly."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It made two errors. First, it treated the dependent's disability benefits as Social Security and added $9,180 to AGI, although those benefits are excluded from the joint return. Second, it subtracted a $1,845 auto-loan-interest deduction whose qualifying conditions aren't met. Correct taxable income is $55,102."
+us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It gave a $2,000 child tax credit for the 23-year-old, but the CTC requires the child to be under 17, so a disabled adult qualifying child gets only the $500 credit for other dependents. It also added the dependent's nontaxable $10,800 of disability benefits to AGI (and mis-added the total as $97,202) and used a $30,000 standard deduction instead of $32,200."
+us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the pre-credit tax of $6,116.24 exactly, then applied no nonrefundable credit, reasoning that a 23-year-old can't get the CTC. It missed the $500 credit for other dependents, which applies because a permanently and totally disabled child of any age is a qualifying child."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the dependent's $10,800 of disability benefits as taxable income on the joint return, raising AGI to $98,102 instead of $87,302. It also used a $31,200 standard deduction and a $24,150 10% bracket instead of 2026's $32,200 and $24,800."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a made-up $10,000 child tax credit that wiped out the whole liability. The household has no CTC-eligible child under 17, and the only nonrefundable credit is the $500 credit for the disabled adult dependent, which leaves $5,616.24 of tax on $55,102 of taxable income."
us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0."
us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC."
us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0."
-us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model derived the exact reference figure four separate times ($82,360 × 6.2% = $5,106.32 plus × 1.45% = $1,194.22 = $6,300.54) and then abandoned it by imputing wages to the Spouse from the Head's $67 hourly rate × 50 hours × 52 weeks = $174,200, violating the instruction to treat unlisted numeric inputs as 0 and misassigning the Head's stated hourly wage to a different person. Its final $12,554.41 matches none of its own candidate totals ($15,554.41, $15,454.41, $14,227.26, $14,232.66), so it submitted a number no calculation in its reasoning produces."
-us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model applied the correct rule and correct base — 6.2% and 1.45% on the head's $82,360 — but truncated each component to whole dollars ($5,106 and $1,194), discarding the $0.54 of cents that the exact products carry, and then mis-added those truncated components: $5,106 + $1,194 = $6,300, not the $6,299 it submitted. The result is $1.54 short of the exact $6,300.54."
-us,scenario_067,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model explicitly extended the FICA base to ""pension/disability income,"" adding the head's $4,942 taxable private pension and the dependent's $10,800 disability benefits to the wage base; neither is wages for Social Security or Medicare purposes under IRC §3121(a), which taxes only remuneration for employment. Its $7,183 is 7.65% of roughly $93,900 — about $11,500 of non-wage income piled onto the $82,360 of actual wages — where 7.65% of $82,360 alone gives the correct $6,300.54."
-us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule and scope — Social Security and Medicare on the head's $82,360 only, no Additional Medicare Tax — and then submitted a number that rule does not produce: $5,727.14 is 6.95% of $82,360, equivalently 7.65% of only $74,865, i.e. a base $7,495 below the wages it named. The combined 7.65% employee rate on $82,360 yields $5,106.32 + $1,194.22 = $6,300.54, so the answer understates by $573.40 through a slipped multiplication rather than any rule error."
-us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($82,360 × 0.062 = $5,106 and × 0.0145 = $1,194, summing to $6,300) and correctly ruled out Additional Medicare Tax, then discarded its own sum in favor of a fabricated bracket, writing ""total approximately 6300-6400, rounded to 6376."" That invented rounding overstates the exact $6,300.54 by $75.46; carrying its own arithmetic to the cent would have produced the reference value."
+us,scenario_067,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model read the prompt's list of four city taxes as complete and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: Adams County's default 1.6% rate on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) gives $1,348.83."
+us,scenario_067,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model treated the NYC/Philadelphia/Kansas City/St. Louis list as complete and answered $0. It never applied the Indiana county income tax that PolicyEngine includes in local_income_tax, which is 1.6% (Adams County default) of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model said Indiana residents owe no local income tax. In fact, every Indiana county levies a county income tax on Indiana AGI. PolicyEngine includes that tax in local_income_tax: 1.6% (Adams County default) × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model correctly saw that Indiana county income tax applies, but $1,800 is an unexplained round number equal to about 2.1% of income. The engine applies Adams County's 1.6% rate to Indiana AGI of $84,302 ($87,302 federal AGI minus $3,000 of exemptions for three people), which gives $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed city taxes and answered $0. It left out the Indiana county income tax that PolicyEngine's local_income_tax includes: 1.6% of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model treated the output as covering only NYC, Philadelphia, Kansas City and St. Louis taxes, so it answered $0. PolicyEngine's local_income_tax also includes Indiana county tax, computed at the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model knew Indiana county income tax exists but decided the prompt's city list excluded it from this output, so it answered $0. PolicyEngine includes the county tax in local_income_tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It skipped the Indiana county income tax in PolicyEngine's local_income_tax, which is 1.6% (Adams County default) of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model read the four named jurisdictions as the only taxes this output covers and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax at 1.6% of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model treated the enumerated city taxes as complete and answered $0. It missed the Indiana county income tax that PolicyEngine includes in local_income_tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed NYC/Philadelphia/Kansas City/St. Louis taxes and answered $0. PolicyEngine also counts Indiana county income tax here: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model treated the four covered city taxes as the only ones in this output and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% (Adams County default) of $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It never applied the Indiana county income tax that PolicyEngine includes in local_income_tax, 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model read the listed city taxes as complete and answered $0. PolicyEngine's local_income_tax also covers Indiana county income tax, which comes to $1,348.83 at Adams County's 1.6% rate on $84,302 Indiana AGI."
+us,scenario_067,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model answered $0 under the reading that only the enumerated city taxes count. It left out the Indiana county income tax in PolicyEngine's local_income_tax: 1.6% of Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model read the prompt as covering only NYC, Philadelphia, Kansas City and St. Louis, so it answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model said living in Indiana triggers no local income tax. In fact, all Indiana counties levy a county income tax on Indiana AGI, and PolicyEngine counts it in local_income_tax: Adams County's 1.6% rate × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county taxes but decided this output variable excluded them, so it answered $0. PolicyEngine's local_income_tax includes the county tax: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model treated the four specified city taxes as the full scope of the output and answered $0. It omitted Indiana county income tax, which PolicyEngine computes as 1.6% of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana's local taxes are not the type this output models and answered $0. PolicyEngine does model Indiana county income tax in local_income_tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed NYC/Philadelphia/Kansas City/St. Louis jurisdictions and answered $0. PolicyEngine also includes Indiana county income tax, computed with the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model concluded the household was outside any recognized separate local-tax jurisdiction and answered $0. Indiana county income tax is part of PolicyEngine's local_income_tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model said PolicyEngine's local income tax covers only NYC, Philadelphia, Kansas City and St. Louis, so it answered $0. The engine's local_income_tax also includes Indiana county income tax: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the complete list and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax: 1.6% of $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model counted only the listed city taxes and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Because no local jurisdiction was named, the model answered $0. PolicyEngine applies its default Indiana county (Adams, 1.6%) to Indiana AGI of $84,302, which gives $1,348.83 of county income tax in local_income_tax."
+us,scenario_067,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model checked only for NYC/Philadelphia/Kansas City/St. Louis taxes and answered $0. It skipped the Indiana county income tax that PolicyEngine includes, 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model read the four enumerated city taxes as the full scope and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed jurisdictions and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax, which is the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model treated the separately requested city taxes as the complete list and answered $0. PolicyEngine also counts Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana out because it is not one of the specified jurisdictions and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana is not covered by this output and answered $0. PolicyEngine includes Indiana county income tax in local_income_tax: 1.6% (Adams County default) × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Because no listed city jurisdiction applied, the model answered $0. It omitted the Indiana county income tax that PolicyEngine includes, 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model treated the listed jurisdictions as complete and answered $0. PolicyEngine's local_income_tax also covers Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model said the output covers only the specified city taxes and answered $0. PolicyEngine includes Indiana county income tax in local_income_tax: Adams County's 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model said no local income taxes apply in Indiana. In fact, every Indiana county imposes a county income tax on Indiana AGI. PolicyEngine includes it in local_income_tax at 1.6% (Adams County default) × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four named city taxes and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county tax but treated it as outside this output because of the city list, so it answered $0. PolicyEngine counts it in local_income_tax: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax, which is 1.6% of $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model acknowledged that Indiana has county taxes but excluded them because of the prompt's city list, so it answered $0. PolicyEngine includes the county tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model treated the four listed city taxes as the full scope and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model read the output as limited to NYC, Philadelphia, Kansas City and St. Louis taxes and answered $0. PolicyEngine also includes Indiana county income tax, computed as 1.6% of Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model counted only the listed city taxes and answered $0. It skipped the Indiana county income tax in PolicyEngine's local_income_tax, 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model recognized that Indiana county income tax applies but answered $0 because no county was given. PolicyEngine handles a missing county by applying its default, Adams County at 1.6%, to $84,302 Indiana AGI, which gives $1,348.83."
+us,scenario_067,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model explicitly decided that Indiana county taxes fall outside this output and answered $0. PolicyEngine's local_income_tax includes them: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the only modeled local taxes and answered $0. PolicyEngine also models Indiana county income tax: Adams County's 1.6% × $84,302 = $1,348.83."
+us,scenario_067,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model said local taxes apply only in the listed cities and answered $0. It omitted the Indiana county income tax that PolicyEngine includes in local_income_tax, 1.6% of $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model first got the correct $6,300.54 from 7.65% of the head's $82,360. It then made up spouse wages by applying the head's $67 hourly rate to the spouse's 50 weekly hours, even though the prompt says unlisted numbers are 0. After trying several inconsistent versions of that guess, it submitted an unexplained $12,554.41 instead of the head-only figure."
+us,scenario_067,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model assumed the spouse had wages that the prompt does not list, even though the spouse's wages are $0. At 7.65%, its $8,500 answer implies about $111,000 of taxed wages, roughly $28,700 more than the head's $82,360. Taxing only the head's wages gives $6,300.54."
+us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the right base and rates but rounded each piece to whole dollars ($5,106 and $1,194). It then added them to $6,299 instead of $6,300 and blamed a wage limit that does not apply at $82,360. The unrounded total is $5,106.32 + $1,194.22 = $6,300.54."
+us,scenario_067,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model said payroll tax applies to pension and disability income, which is wrong. FICA applies only to wages, so the $4,942 taxable pension and the dependent's $10,800 disability benefits are outside the base. Its $7,183 is about $882 too high, which equals 7.65% of roughly $11,500 of non-wage income added to the head's $82,360."
+us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly said payroll tax applies only to the head's $82,360 of wages and that no Additional Medicare Tax is due. Its $5,727.14 is still $573.40 short of the correct $5,106.32 + $1,194.22 = $6,300.54. That figure equals 7.65% of only about $74,864 of wages, which shows an arithmetic error in combining the Social Security and Medicare pieces."
+us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $5,106 Social Security and $1,194 Medicare on the head's $82,360. It then described the total as a range of $6,300 to $6,400 and arbitrarily submitted $6,376 instead of adding the two pieces to get $6,300.54."
us,scenario_067,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as categorically SSI-eligible on disability and computed (994 − 880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, as it does for the head (60) and spouse (58), so no member passes the aged/blind/disabled gate and SSI is $0 before the income test."
us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the 23-year-old as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and for both adults, so SSI's categorical test fails for every member and the benefit is $0."
@@ -5537,79 +5686,81 @@ us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Treate
us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and computed ($994 − $880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and both adults, so SSI's categorical requirement fails for everyone and SSI is $0."
us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine evaluates is_ssi_aged_blind_disabled=False for every member, so SSI is $0 before any income or deeming step."
us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. On top of that, it used an invented $1,913/month FBR, well above the 2026 individual rate of $994 and the couple rate of about $1,491, and skipped the $20 general exclusion, which gives $22,956 − $10,800 = $12,156."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It invented a $500 extra exemption for the disabled dependent, which gave $3,500 of exemptions instead of $3,000. It then applied a guessed 2.9% rate instead of 2.95% and subtracted unspecified 'minor deductions' to reach $2,295. The correct figure is $84,302 × 2.95% = $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income after subtracting the $28,700 federal standard deduction, but Indiana starts from federal AGI and allows no standard deduction. It also used a 3.15% rate, invented health-premium and medical credits, and then inflated the result to $4,168 with no computation. It should have used $1,000 exemptions for three people and the 2.95% rate."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It gave the 23-year-old dependent a $1,500 exemption instead of the $1,000 dependent exemption, for $3,500 total. It then used a 'projected' 3.12% rate instead of Indiana's 2026 rate of 2.95%. The correct tax is 2.95% of $84,302 = $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"Its reasoning computed the correct $84,302 × 2.95% = $2,486.91. It then switched to a nonexistent $1,500 'disabled dependent' additional exemption, but that exemption requires a child under 19 or a full-time student under 24, so disability does not qualify. It finally submitted $2,620, which matches neither of its own calculations."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted unspecified 'additional dependent-related amounts' beyond the three $1,000 exemptions. It also applied a 2.9% rate instead of Indiana's 2026 flat rate of 2.95%, which produced $2,410 instead of $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,942 pension under an Indiana 'age 60+ pension deduction' that does not exist, and it added a $1,500 disabled-dependent exemption, for $4,500 of exemptions. It also used the old 3.05% rate. The correct calculation keeps the pension in Indiana AGI and taxes $84,302 at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It claimed a $1,500 dependent exemption plus a $1,500 additional qualifying-child exemption, for $5,000 total, but the 23-year-old non-student only gets the $1,000 dependent exemption. It used the old 3.05% rate and then pushed its own $2,510 up to $2,650 with no computation."
-us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put the dependent's $10,800 of disability benefits into Indiana AGI ($98,102) and subtracted a fictitious $4,000 MFJ standard deduction; Indiana has no standard deduction. It then used a 3.0% rate instead of 2.95%. The correct base is $87,302 minus $3,000 of exemptions."
-us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It derived the correct Indiana taxable income of $84,302 ($87,302 AGI minus three $1,000 exemptions). It then applied a 3.0% rate instead of Indiana's 2026 flat rate of 2.95%, overstating the tax by $42.15."
-us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It cited the correct 2.95% rate but subtracted a 'standard deduction', which Indiana does not have. Its $2,226.48 implies a taxable base of about $75,474, or about $11,800 of deductions, instead of the $3,000 of personal and dependent exemptions that leave $84,302."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It added a $1,500 additional exemption for a 'disabled child'. Indiana's additional dependent-child exemption applies only to children under 19 or full-time students under 24, and disability does not substitute. It then applied 3.0% instead of 2.95% to $82,802. The two errors partly offset each other, giving $2,484.06 instead of $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied the outdated 3.05% rate. Its $2,489 implies a base of about $81,600, which matches no Indiana exemption structure. The correct calculation is 2.95% of $84,302 ($87,302 AGI minus three $1,000 exemptions)."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,age_disability,False,"It used the correct 2.95% rate but subtracted about $4,500 of exemptions. That total includes a $1,500 additional dependent exemption the 23-year-old non-student does not qualify for, because it requires a child under 19 or a full-time student under 24. Exemptions are $3,000, leaving $84,302 of taxable income."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed the correct $84,302 base ($87,302 minus three $1,000 exemptions) but applied the outdated 3.05% rate instead of Indiana's 2026 flat rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation, and its $5,382 is more than double the correct $2,486.91. That implies an effective rate of about 6.4% on the $84,302 base, far above Indiana's 2.95% flat state rate, which consistent with adding county tax that the question excludes."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It derived the correct $84,302 taxable income from $87,302 AGI minus three $1,000 exemptions. It then applied a 3.0% rate instead of Indiana's 2026 flat rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,529.06 is exactly 3.0% of the correct $84,302 base, so it applied a 3.0% flat rate instead of Indiana's 2026 rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It skipped all Indiana personal and dependent exemptions ($3,000) and taxed the full $87,302 AGI. It also used a 2.75% rate instead of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It dropped the dependent's $1,000 exemption on the theory that the 23-year-old's $10,800 of disability benefits disqualify them as a dependent. The household states this person is a dependent, so three exemptions apply. It also used 2.9% instead of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation and referred to a 'standard deduction' Indiana does not have. Its $3,501 is about 4.0% of total AGI, far above the 2.95% flat rate applied to $84,302 after three $1,000 exemptions."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted standard deductions plus mortgage-interest and medical deductions, but Indiana starts from federal AGI and allows neither a standard deduction nor federal itemized deductions. Its $2,183 implies a base of about $74,000 instead of $84,302 at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,age_disability,False,"It used the correct 2.95% rate but subtracted $4,500 of exemptions, which includes a $1,500 additional dependent exemption. That exemption requires a child under 19 or a full-time student under 24, which this 23-year-old non-student dependent is not. Exemptions are $3,000, so taxable income is $84,302."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It said it subtracted $3,000 of exemptions, but its $2,619.06 is exactly 3.0% of the full $87,302 AGI, so it applied no exemptions. It also used 3.0% instead of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,age_disability,False,"It added a $1,500 'qualifying-child' exemption for the 23-year-old disabled dependent, which brought exemptions to $4,500. Indiana's additional dependent-child exemption requires a child under 19 or a full-time student under 24, so only the $1,000 dependent exemption applies and taxable income is $84,302."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It correctly excluded the disability benefits and used 2.95%, but it subtracted $4,500 of exemptions. That includes the $1,500 additional dependent-child exemption, which is limited to children under 19 or full-time students under 24, and this 23-year-old non-student is neither. Exemptions are $3,000."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but a taxable income of $83,802, which means $3,500 of exemptions. It gave the dependent $1,500 instead of the $1,000 dependent exemption Indiana allows a 23-year-old non-student."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no computation and cited unspecified 'deductions'. Its $2,200 implies a base of about $74,600 at 2.95%, which means roughly $12,700 of deductions. Indiana allows only $3,000 of exemptions here and no standard deduction, leaving $84,302."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It computed the correct $84,302 base (federal AGI minus three $1,000 exemptions) but applied the outdated 3.05% rate instead of Indiana's 2026 rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly subtracted three $1,000 exemptions from $87,302 but applied a 3% flat rate instead of Indiana's 2026 rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.7,llm_error,age_disability,False,"It used the correct 2.95% rate but added a $1,500 additional dependent exemption, for $4,500 total. That exemption applies only to children under 19 or full-time students under 24, so the 23-year-old non-student dependent does not qualify and taxable income is $84,302."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied the rate to the full $87,302 AGI without subtracting Indiana's $1,000 personal exemptions for head, spouse, and dependent. It also used 2.9% instead of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It subtracted only the two filers' $1,000 exemptions, omitting the dependent's $1,000 exemption, and applied 2.9% instead of Indiana's 2026 rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $2,486.91 reference."
-us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,age_disability,False,"It used 2.95% but claimed a $1,500 dependent exemption plus a $1,500 'additional disabled-child' exemption, for $5,000 total. Indiana gives this dependent only the $1,000 dependent exemption, because the additional exemption requires age under 19 or full-time student status under 24."
-us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It started from a wrong AGI of $97,202, which pulls in the dependent's disability benefits (and even then mis-adds them). It subtracted a nonexistent $3,800 Indiana standard deduction and applied the outdated 3.05% rate. The correct base is $87,302 minus $3,000 of exemptions, taxed at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,age_disability,False,"It claimed a $1,500 dependent exemption plus a $1,500 disabled-dependent additional exemption, for $5,000 total. The 23-year-old non-student gets only the $1,000 dependent exemption. It also applied 2.9% instead of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It reached the right $84,302 base, but only by calling the $3,000 a standard deduction. It then applied the outdated 3.05% rate and subtracted an invented $100 dependent credit and about $374 of fabricated nonrefundable credits. Indiana has no such credits for this household."
-us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of disability benefits to Indiana AGI, for $98,102, but those benefits are not part of the filers' AGI. It also used the outdated 3.05% rate instead of 2.95% on the correct $84,302 base."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model invented a $500 disabled-dependent exemption, giving $3,500 instead of $3,000. It applied a ~2.9% rate instead of the 2026 rate of 2.95%, then cut the result to $2,295 with unexplained 'minor deductions'. The correct figure is $84,302 × 2.95% = $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the federal $28,700 standard deduction even though Indiana starts from AGI and has no standard deduction. It used a 3.15% rate and invented health-premium, medical and senior credits. Its final $4,168 contradicts its own interim $1,356. Indiana allows only $3,000 of exemptions and taxes the remaining $84,302 at 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model stated a 3.05% rate, which is the 2024 rate rather than the 2026 rate of 2.95%. Its $4,500 answer works out to about 5.3% of the ~$84,302 post-exemption base, so it did not even apply the rate it stated. The correct tax is $84,302 × 2.95% = $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model gave the 23-year-old non-student dependent a $1,500 exemption instead of the $1,000 one, for $3,500 total. It also used a made-up 'projected' 3.12% rate instead of the statutory 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly computed $84,302 × 2.95% = $2,486.91. It then dropped that result to add a $1,500 'disabled dependent' exemption, which does not exist in Indiana. Finally it submitted $2,620, which matches neither of its own calculations."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used a ~2.9% rate instead of 2.95%. It also added unspecified 'additional dependent-related amounts' on top of the correct $3,000 of exemptions, which shrank the base below $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $4,942 private pension as an age-60 pension deduction, but Indiana has no such deduction for private pensions. It added a nonexistent $1,500 disabled-dependent exemption and used the 3.05% rate instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model claimed $5,000 of exemptions, giving the dependent $1,500 plus an extra $1,500 qualifying-child amount. A 23-year-old non-student qualifies only for the $1,000 exemption. It used 3.05% instead of 2.95% and then raised its own $2,510 figure to $2,650 with no basis."
+us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model put the dependent's $10,800 disability benefits into the filers' AGI ($98,102 instead of $87,302). It subtracted an invented $4,000 MFJ standard deduction, which Indiana does not have, and used 3.0% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model got the $84,302 taxable base right but applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%. That overstated the tax by about $42."
+us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model used the correct 2.95% rate but subtracted a 'standard deduction' on top of the exemptions, which Indiana does not allow. Its $2,226.48 implies a base of about $75,474 instead of $84,302 (AGI minus only the $3,000 of exemptions)."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model added a $1,500 disabled-child exemption, but the 23-year-old non-student dependent qualifies only for the $1,000 exemption. That cut the base to $82,802. It also used 3.0% instead of the 2026 rate of 2.95%, so its total landed near the reference only because the two errors partly cancel."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied the 2024 rate of 3.05% instead of the 2026 rate of 2.95%. Its $2,489 implies a taxable base of about $81,600 instead of $84,302, so it also over-subtracted exemptions."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $4,500 of exemptions, adding a $1,500 additional dependent exemption. That exemption applies only to children under 19 or full-time students under 24, not to this 23-year-old non-student. The base should be $84,302 rather than $82,802."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model got the $84,302 taxable base right but applied the 2024 rate of 3.05% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no calculation. Its $5,382 is about 6.2% of AGI, more than double Indiana's 2.95% flat rate applied to the $84,302 base."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got the $84,302 taxable base right but used the 2025 rate of 3.0% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,529.06 equals $84,302 × 3.0%, so it used the 2025 rate instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model took no exemptions, taxing the full $87,302 instead of subtracting $3,000 for three people. It also used a 2.75% rate instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"The model denied the dependent exemption because of the dependent's $10,800 in disability benefits. Those benefits are not taxable gross income, so the dependent still counts and the third $1,000 exemption applies. It also used 2.9% instead of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model cited a standard deduction Indiana does not have. Its $3,501 is about 4.15% of the $84,302 base, far above the 2.95% flat rate, which gives $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction and mortgage-interest and medical deductions, but Indiana starts from federal AGI and allows only the $3,000 of personal exemptions here. Its $2,183 implies a base of about $74,000 instead of $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed $4,500 of exemptions, adding a $1,500 extra dependent amount. The 23-year-old non-student dependent is not under 19 and not a student under 24, so only $3,000 applies and the base is $84,302 × 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $2,619.06 equals $87,302 × 3.0%, so it applied the rate to AGI without subtracting the $3,000 of exemptions it described. It also used 3.0% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model added a $1,500 qualifying-child exemption for the 23-year-old non-student, for $4,500 total instead of $3,000. That exemption is limited to dependents under 19 or full-time students under 24."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used $4,500 of exemptions, including an extra $1,500 dependent amount the 23-year-old non-student does not qualify for. That cut the base from $84,302 to $82,802."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model gave the dependent a $1,500 exemption instead of the $1,000 personal exemption, for $3,500 total. That taxed $83,802 instead of $84,302 at 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave no calculation beyond 'after deductions'. Its $2,200 implies about $74,600 of taxable income at 2.95%, so it subtracted deductions Indiana does not allow. Only the $3,000 of exemptions comes off the $87,302 AGI."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model got the $84,302 base right but applied the 2024 rate of 3.05% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model got the $84,302 base right but applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model added a $1,500 additional dependent exemption on top of three $1,000 exemptions. The 23-year-old non-student dependent does not qualify for it, so the base is $84,302, not $82,802."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted none of the three $1,000 personal exemptions. It taxed the full $87,302 at 2.9% instead of $84,302 at 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model took exemptions for only the two filers and left out the $1,000 exemption for the dependent. It also used 2.9% instead of the 2026 rate of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so its answer could not be parsed."
+us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model claimed $5,000 of exemptions, giving a $1,500 dependent exemption and an invented $1,500 disabled-child exemption. A 23-year-old non-student dependent gets only the $1,000 exemption, so the base is $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model inflated federal AGI to $97,202 by counting the dependent's disability benefits. It subtracted an invented $3,800 Indiana standard deduction and used 3.05% instead of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used $5,000 of exemptions, giving the dependent $1,500 plus a nonexistent $1,500 disabled-dependent exemption, where only $1,000 applies. It also used 2.9% instead of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model got the $84,302 base right but used 3.05% instead of 2.95%. It then subtracted an invented $100 dependent credit and $374 of made-up 'estimated nonrefundable credits'. No nonrefundable Indiana credits apply."
+us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model added the dependent's $10,800 disability benefits to the filers' AGI. The correct AGI is $87,302 (wages plus pension). It also used 3.05% instead of the 2026 rate of 2.95%."
us,scenario_067,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $73,000 of educational assistance as employer §127 assistance and added the $67,750 above the $5,250 cap to wages, which pushed taxable income to $85,734 instead of $17,983.53. It then submitted $8,258, a number that matches none of its own arithmetic (it computed $13,573). With the correct wage-only base, its own $16,100 deduction and $12,400/$50,400 brackets give $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It used the 2024 $14,600 standard deduction, added $5,250 of educational assistance to income, and prorated wages over 40 weeks. It then zeroed the tax by subtracting the EITC, which is a refundable credit and so is excluded from this measure. A childless 31-year-old with $34,084 of earnings is also fully phased out of the EITC. The correct figure is $1,910.02 on $17,983.53 of taxable income."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 $5,250 cap to wages, making AGI $101,834 when it is really the $34,083.53 of wages alone. It also used stale bracket edges ($12,150/$49,350) in place of 2026's $12,400/$50,400."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Its 2026 parameters were correct ($16,100 deduction, $12,400/$50,400 brackets), but it turned the separately listed $73,000 of educational assistance into $67,750 of taxable wages. That set taxable income at $85,734 instead of $17,983.53 and produced $13,573 instead of $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly excluded the educational assistance, applied the $16,100 deduction and the $12,400 10% bracket edge, and reached about $1,910 in its own reasoning. It then submitted an unsupported $2,372, which no step of its calculation produces."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages, making AGI $101,834 when it is really $34,083.53. It compounded this with a $15,750 standard deduction and invented bracket edges ($12,200/$46,600) in place of 2026's $16,100 and $12,400/$50,400."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It first taxed $67,750 of educational assistance, then switched to a wage-only base of $17,984 taxable income and computed $1,915 using a stale $12,150 10% bracket edge. It then reported an unexplained $1,750 that none of its calculations produce. The correct wage-only computation with the $12,400 edge is $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the entire $73,000 of educational assistance to wages, making AGI $107,084 when it is really $34,083.53. It also used a $15,700 deduction and $12,500 bracket edge instead of 2026's $16,100 and $12,400, and so taxed $91,384 in the 22% bracket in place of $17,983.53 of taxable income."
-us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made two income errors: it subtracted the $8,389 ESI premium from the stated gross wages, and it added $67,750 of educational assistance above the §127 cap. That gave an AGI of $93,445 instead of $34,083.53. It also used a $15,400 deduction instead of $16,100."
-us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages, making AGI $101,834 when it is really $34,083.53. It then applied pre-TCJA law, with a $7,900 deduction, a $5,050 personal exemption and 15%/25% brackets. For 2026 the law is the permanent $16,100 deduction with 10%/12% brackets."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, giving AGI of $25,695 when it is really $34,083.53. It also assumed a 2026 TCJA sunset ($7,600 deduction + $5,100 exemption, 15% bracket), but the $16,100 standard deduction and 10%/12% brackets apply."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used a $15,200 deduction and taxed everything at 10% ($1,888). It then cut the tax to $139 by applying a nonrefundable Child Tax Credit, but the single filer has no qualifying child, so no nonrefundable credit applies and tax stays at $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from gross wages (AGI $25,695 when it is really $34,083.53). It also assumed a TCJA sunset with about $14,250 of deduction plus exemption, but the $16,100 standard deduction applies, leaving $17,983.53 taxed at 10%/12%."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to income and subtracted the $8,389 ESI premium, giving AGI of $98,695 when it is really $34,083.53. It then applied sunset-era law ($8,300 deduction, $5,050 exemption, 15%/25% brackets) in place of the 2026 $16,100 deduction and 10%/12% brackets."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation for $0. Wages of $34,083.53 less the $16,100 standard deduction leave $17,983.53 of taxable income and $1,910.02 of tax with no nonrefundable credits. A zero answer means it treated this wage earner as having no positive liability, which the standard deduction alone never produces here."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied an $8,300 deduction, a $5,100 personal exemption and a 15% bracket. For 2026 the permanent $16,100 standard deduction, no exemption, and 10%/12% brackets apply, giving $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation lists the educational assistance among the tax inputs, and $11,116 is reachable only by taxing tens of thousands of dollars above the $34,084 of wages. It counted the educational assistance as taxable income, but only the $34,083.53 of wages is taxed: $17,983.53 of taxable income and $1,910.02 of tax."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed wages only but used stale pre-OBBBA 2025 parameters: a $15,000 standard deduction and an $11,925 10% bracket edge. The 2026 values are $16,100 and $12,400, so it overstated taxable income by $1,100 and reached $2,051.58."
-us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages, making taxable wages $101,834 instead of $34,083.53. It also used 2024 parameters ($14,600 deduction, $11,600/$47,150 brackets) as a proxy for 2026."
-us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the correct income base and $16,100 deduction but set the top of the 10% bracket at $12,200 instead of 2026's $12,400. That taxes $200 at 12% instead of 10% and gives $1,914 instead of $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It claimed that $34,084 of wages is too low to owe tax after the standard deduction. The 2026 single deduction is only $16,100, which leaves $17,983.53 of taxable income and $1,910.02 of tax."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It declared the liability negative or zero after the standard deduction. The $16,100 deduction leaves $17,983.53 taxable, which produces $1,910.02 at the 10%/12% rates with no nonrefundable credits."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a projected post-sunset regime (about $13,650 of deduction plus personal exemption, a 15% second bracket). For 2026 the permanent $16,100 standard deduction and 10%/12% brackets apply, giving $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the $5,250 exclusion as taxable income. That pushed taxable income to about $85,734 and tax to $13,573.48, when only the $34,083.53 of wages is taxed ($1,910.02)."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, making income $107,084 when it is really $34,083.53. After the correct $16,100 deduction that taxed $90,984 instead of $17,983.53."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $19,000 implies a stale deduction of roughly $15,000 instead of the 2026 $16,100. That overstates taxable income by about $1,000 and gives $2,058 instead of $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025-like parameters, a $15,000 standard deduction and an $11,925 10% bracket edge, instead of 2026's $16,100 and $12,400. That produced $19,084 of taxable income and $2,052 of tax."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules: an $8,450 deduction, a $5,300 personal exemption and a 15% bracket. For 2026 the permanent $16,100 standard deduction and 10%/12% brackets apply to the $34,083.53 of wages."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the correct base and $16,100 deduction but put the top of the 10% bracket at $12,200 instead of 2026's $12,400. That produced $1,914.08 instead of $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages (AGI $25,695 when it is really $34,083.53). It also used a projected post-TCJA $8,527 deduction with 15% brackets in place of the 2026 $16,100 deduction and 10%/12% brackets."
-us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so no answer was available to score."
-us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and a $12,200 10% bracket edge instead of 2026's $16,100 and $12,400. That overstated taxable income at $18,334 and gave $1,956."
-us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and a $12,150 10% bracket edge instead of 2026's $16,100 and $12,400. That gave $18,334 of taxable income and $1,957.08 of tax."
-us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It subtracted the EITC to reach $0 taxable income. The EITC is a refundable credit and never reduces taxable income or this pre-refundable-credit measure, and a childless filer with $34,084 of earnings is fully phased out of it. Wages less the $16,100 deduction leave $17,983.53 taxable and $1,910.02 of tax."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of the educational assistance (the amount above the §127 cap) to taxable income, but the assistance is excluded entirely and AGI is wages alone. It then submitted $8,258, which does not match its own $13,573 calculation. The correct figure is $17,983.53 taxed at 10%/12%, which is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It used the outdated $14,600 standard deduction and subtracted the $5,250 educational exclusion from wages. It then let the EITC wipe out the tax, but the EITC is refundable and does not reduce tax before refundable credits. With wage-only AGI, the $16,100 deduction leaves $17,983.53 of taxable income and $1,910.02 of tax."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"Its reasoning reached roughly $18,000 of taxable income, a positive tax, and no nonrefundable credits, yet it submitted $0. Carrying the $17,983.53 of taxable income through the 10%/12% brackets gives $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated $67,750 of the educational assistance as taxable wages, raising AGI to $101,834 and pushing income into the 22% bracket. It also used wrong 2026 bracket edges ($12,150/$49,350). The assistance is excluded entirely, so taxable income is $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added $67,750 of the educational assistance (above the §127 cap) to AGI. The assistance is excluded entirely, so AGI is $34,083.53 and taxable income is $17,983.53, which never reaches the 22% bracket."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning used the correct framework: wages only, the $16,100 deduction, $17,984 of taxable income, and a 10% bracket up to $12,400. That gives about $1,910, but it then submitted an unexplained $2,372 instead of carrying that figure through."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included $67,750 of the educational assistance in AGI, which should be wages only. It also used a stale $15,750 standard deduction and $12,200/$46,600 bracket edges in place of the 2026 values ($16,100; $12,400/$50,400)."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It eventually reached the correct $17,984 of taxable income but used a 10% bracket top of $12,150 instead of $12,400, giving $1,915. It then submitted an arbitrary $1,750 instead of the $1,910.02 that the correct brackets produce."
+us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the full $73,000 of educational assistance as taxable income (AGI $107,084), but the assistance is excluded and AGI is $34,083.53. It also used a $15,700 standard deduction instead of the 2026 single amount of $16,100."
+us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 of ESI premiums from the reported gross wages and added $67,750 of educational assistance to AGI. Both adjustments are wrong: AGI is just the $34,083.53 in wages. It also used a $15,400 standard deduction in place of $16,100."
+us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to AGI and applied pre-TCJA law: a $7,900 standard deduction, a $5,050 personal exemption, and 15%/25% rates. Under 2026 law, the $16,100 standard deduction and the 10%/12% brackets apply to wage-only taxable income of $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using a $7,600 standard deduction plus a $5,100 personal exemption and a 15% second bracket. In fact, 2026 has a $16,100 standard deduction, no personal exemption, and a 12% second bracket. It also wrongly subtracted the $8,389 of ESI premiums from the stated gross wages."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It applied 10% to all of its $18,884 of taxable income, ignoring the 12% bracket above $12,400. It then subtracted a nonexistent Child Tax Credit, although the filer has no dependents and no nonrefundable credits. With the $16,100 deduction, tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters, about $14,250 in standard deduction plus personal exemption, instead of the 2026 $16,100 standard deduction. It also subtracted the $8,389 of ESI premiums from gross wages, which understated taxable income at $11,445 instead of $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to AGI and subtracted the ESI premiums, then applied pre-TCJA law ($8,300 deduction, $5,050 exemption, 15%/25% rates). The correct AGI is wages alone, taxed after the $16,100 deduction at 10%/12%."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave $0 with no supporting derivation. Wage-only AGI of $34,083.53 minus the $16,100 standard deduction leaves $17,983.53 of taxable income, which produces $1,910.02 of tax with no offsetting nonrefundable credits."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026, using an $8,300 standard deduction, a $5,100 personal exemption, and a 15% second bracket. Current 2026 law instead has a $16,100 standard deduction, no exemption, and a 12% rate above $12,400."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $11,116 implies taxable income of about $75,000, which only happens if most of the $73,000 educational assistance is added to income. The assistance is excluded, so taxable income is $17,983.53 and tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used stale parameters, a $15,000 standard deduction and an $11,925 top for the 10% bracket, instead of the 2026 values of $16,100 and $12,400. As a result, it overstated taxable income ($19,084 instead of $17,983.53) and pushed more of it into the 12% bracket."
+us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to taxable wages, but the assistance is excluded. It also used 2024 parameters (a $14,600 deduction and $11,600/$47,150 brackets) instead of the 2026 values ($16,100; $12,400/$50,400)."
+us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It correctly reached $17,984 of taxable income but set the top of the 2026 single 10% bracket at $12,200 instead of $12,400. That taxes $200 too much at 12% and gives $1,914 instead of $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the income is too low to owe tax after the standard deduction, but $34,083.53 of wages exceeds the $16,100 deduction by $17,983.53. That amount is taxed at 10%/12%, giving $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It said tax after the standard deduction was negative or zero, but wages of $34,083.53 minus the $16,100 deduction leave $17,983.53 of positive taxable income. That produces $1,910.02 of tax with no nonrefundable credits to offset it."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, using a combined standard deduction and personal exemption of about $13,650 and a 15% second bracket. The 2026 parameters are a $16,100 standard deduction, no exemption, and a 12% rate above $12,400."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance (above the $5,250 exclusion) in taxable income, raising AGI to $101,834. The assistance is excluded, so AGI is the $34,083.53 in wages and taxable income is $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, producing $107,084 of income. The assistance is excluded, so only the $34,083.53 in wages less the $16,100 deduction is taxable."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $19,000 implies a standard deduction of about $15,000 instead of the 2026 single amount of $16,100. Its bracket edges are also pre-2026, which overstates both taxable income and the amount taxed at 12%."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025-style parameters, a $15,000 standard deduction and an $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400, which give $17,983.53 of taxable income and $1,910.02 of tax."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: an $8,450 standard deduction, a $5,300 personal exemption, and a 15% second bracket. 2026 law instead has a $16,100 standard deduction, no exemption, and a 12% rate above $12,400."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly reached $17,984 of taxable income but used $12,200 as the top of the 2026 single 10% bracket instead of $12,400. That overstates tax by $4, giving $1,914.08 instead of $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,527 standard deduction and a 15% second bracket instead of the 2026 $16,100 deduction and 12% rate. It also wrongly subtracted the $8,389 of ESI premiums from the stated gross wages."
+us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,750 instead of the 2026 amount of $16,100, and set the 10% bracket top at $12,200 instead of $12,400. Both errors overstated taxable income and the amount taxed at 12%."
+us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and a $12,150 top for the 10% bracket instead of the 2026 single values of $16,100 and $12,400. That produced $18,334 of taxable income instead of $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It subtracted the EITC to drive tax to $0, but the EITC is refundable and is excluded from tax before refundable credits. In any case, a filer with $34,084 of wages and no children gets no EITC. Wages minus the $16,100 deduction leave $17,983.53 of taxable income and $1,910.02 of tax."
us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly projected the temporary American Rescue Plan expansion of the childless EITC into 2026 and therefore used a phaseout endpoint near $33,000–$34,000. Under 2026 law, the temporary expansion is not in effect, and $34,084 exceeds the applicable childless EITC income limit, yielding zero EITC."
us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated wages of $34,084 and a one-person household as qualifying for EITC without applying the childless single-filer phaseout endpoint. At that income the childless EITC is fully phased out, and no other refundable credit applies."
us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract."
@@ -5629,86 +5780,88 @@ us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_068,payroll_tax,inkling,llm_error,state_local_rule,False,"The model got both FICA components right on the full $34,084 wage base and then added a 0.45% Maryland FAMLI employee contribution of $153.38. Maryland's FAMLI employee contributions are not in effect for 2026 and are not part of PolicyEngine's payroll_tax, which comprises only employee Social Security, Medicare, and Additional Medicare Tax."
us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive rule or computation was attempted. The required derivation is 6.2% plus 1.45% on the full $34,084 of wages, with no ESI offset, no educational assistance in the base, and no Maryland employee payroll contribution."
us,scenario_068,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct base and rates but truncated each component to whole dollars ($2,113 Social Security and $494 Medicare) and then submitted $2,600 rather than the $2,607 those components sum to. The rule application was right; the reported total contradicts the model's own arithmetic by $7."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the Section 127 limit to AGI, getting $101,834, but Maryland AGI is only the $34,083.53 of wages. It then used the repealed capped standard deduction ($2,700 instead of the flat $3,350), and its inflated AGI led it to cut the exemption to $1,600. With $34,083.53 of AGI, the $3,350 deduction and the full $3,200 exemption, the schedule gives $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer insurance premium from the listed gross wages and called taxable income 'minimal or zero' without applying the rate schedule. Maryland AGI is the full $34,083.53 of wages. Even its own $25,695 AGI minus $3,900 and $3,200 leaves about $18,600 of taxable income, which Maryland's 2%–4.75% schedule taxes at about $800, not $0."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Its reasoning produced $4,554 (after adding $67,750 of educational assistance to AGI) and $1,336 (wage-only AGI with the old $2,700 standard-deduction cap), but it submitted $1,029, which matches neither. The correct path uses wage-only AGI of $34,083.53, the flat $3,350 standard deduction and the $3,200 exemption, giving $27,533.53 of taxable income and $1,255.34 of tax."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used the repealed 15%-of-income standard deduction with a $2,700 cap instead of Maryland's flat $3,350 single deduction, getting $28,184 of taxable income and $1,286 of tax. It then added an invented 'personal exemption credit adjustment' to reach $1,308. The $3,200 exemption is subtracted from income; it is not a separate credit."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the old capped standard deduction ($2,700) instead of the flat $3,350, which gave $28,184 of taxable income. Its $1,301 also misapplies the schedule, because $90 + 4.75% × $25,184 = $1,286.24. The correct taxable income of $27,533.53 is taxed at $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the $5,250 Section 127 limit in Maryland AGI ($101,834), but Maryland AGI is only the $34,083.53 of wages. It also used a $2,500 cap from the repealed 15% formula instead of the flat $3,350 standard deduction. Together these pushed taxable income to $96,134 and tax to $4,514."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $2,550 capped standard deduction from the repealed 15%-of-income formula instead of the flat $3,350. It also misapplied its own schedule: $28,334 of taxable income gives $1,293.37 under the 2%/3%/4%/4.75% brackets, not $1,050."
-us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, getting $107,084 of AGI, but Maryland AGI is only the $34,083.53 of wages. It also used a $2,400 standard-deduction cap and a phased-down $3,300 exemption instead of the flat $3,350 deduction and full $3,200 exemption, which pushed income into the 5% bracket."
-us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It built a $93,445 AGI by adding $67,750 of educational assistance above the Section 127 cap and subtracting the $8,389 employer insurance premium from wages. Maryland AGI is just the $34,083.53 of listed wages, with neither adjustment. Its $5,980 of deductions is also below the $6,550 total of the flat $3,350 standard deduction and the $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the Section 127 limit in AGI ($101,834), used a $2,400 standard-deduction cap instead of the flat $3,350, and left out the $3,200 personal exemption entirely. The correct taxable income is $34,083.53 − $3,350 − $3,200 = $27,533.53."
-us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its wage-only AGI and $3,200 exemption were correct, but it used a $2,800 standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $550 ($28,084 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It cut AGI to $25,695 by subtracting the $8,389 employer-sponsored insurance premium from the listed gross wages, but Maryland AGI is the full $34,083.53. It also used the repealed $2,550 capped standard deduction instead of the flat $3,350."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $1,133 matches Maryland taxable income of about $24,960, which is about $2,575 below the correct $27,533.53 ($34,083.53 of AGI minus the flat $3,350 standard deduction and the $3,200 exemption). So it overstated deductions by about $2,575."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It subtracted the $8,389 employer insurance premium from wages (AGI of $25,695 instead of $34,083.53) and used a $2,750 standard deduction instead of the flat $3,350. It then added about 3% of county income tax ($592.35) to the state figure, even though this output excludes local income tax."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It built a $98,695 AGI by adding the full $73,000 of educational assistance and subtracting the $8,389 insurance premium, but Maryland AGI is only the $34,083.53 of wages. It also used a $2,550 capped standard deduction instead of the flat $3,350, and a $60 base on the first $3,000 instead of $90."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $997 matches taxable income of about $22,095, which is about $5,440 below the correct $27,533.53. That gap fits an income base shrunk below the listed wages (for example, by subtracting the $8,389 insurance premium) instead of $34,083.53 of AGI minus the flat $3,350 standard deduction and the $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,550 'maximum' standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That left $28,334 of taxable income instead of $27,533.53 and tax of $1,293.37 instead of $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,786 matches taxable income of about $101,770 under Maryland's schedule (4.75% up to $100,000, then 5%), which is only reachable by counting the $73,000 of educational assistance as income. Only the $34,083.53 of wage AGI is taxable, giving $27,533.53 of taxable income and $1,255.34 of tax."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,700 standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That produced $28,184 of taxable income instead of $27,533.53 and tax of $1,286.24 instead of $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $2,750 standard deduction and a $3,300 exemption instead of the flat $3,350 and $3,200, getting $28,034 of taxable income. It then misapplied the rate schedule: $90 + 4.75% × $25,034 = $1,279.12, not $1,453.28."
-us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a $3,200 standard deduction instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $150 ($27,684 instead of $27,533.53) and tax by about $7, and it then rounded to $1,262."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions and exemptions fully offset the wages. Maryland's $3,350 standard deduction and $3,200 exemption total only $6,550 against $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It floored the tax at $0 without subtracting actual deductions or applying the rate schedule. $34,083.53 of AGI minus the $3,350 standard deduction and the $3,200 exemption leaves $27,533.53 of taxable income, taxed at $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It estimated a $2,900 standard deduction from the repealed capped formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $450 ($27,984 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It started from a $101,834 AGI that includes $67,750 of educational assistance above the Section 127 limit, but Maryland AGI is only the $34,083.53 of wages. That inflated the tax to $4,478.99."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It projected a $2,850 maximum standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 of taxable income equals $107,084 − $2,550 − $3,200, so it counted the full $73,000 of educational assistance as income. Maryland AGI is only the $34,083.53 of wages. It also used the repealed $2,550 cap instead of the flat $3,350 standard deduction."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It estimated a $2,900 standard deduction from the repealed capped formula instead of Maryland's flat $3,350 single deduction. That left $27,984 of taxable income instead of $27,533.53 and tax of $1,276.74 instead of $1,255.34."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It indexed the standard deduction up to $3,450 for 2026 instead of using the flat $3,350 single amount. That understated taxable income by $100 ($27,434 instead of $27,533.53) and tax by about $4.75."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave only a rounded guess ('approximately 1400'). $1,400 matches taxable income of about $30,580, which implies only about $3,500 of total deductions and exemptions instead of the $6,550 from the flat $3,350 standard deduction plus the $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $2,350 standard deduction from the repealed 15%-of-income cap instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $1,000 ($28,534 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $2,600 standard-deduction maximum from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $750 ($28,284 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It treated the standard deduction as 15% of AGI capped at $2,850, but Maryland now allows a flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer insurance premium from the listed gross wages (AGI of $25,695 instead of $34,083.53) and used a $2,400 capped standard deduction instead of the flat $3,350. Together these understated taxable income by about $7,440."
-us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used a $2,650 standard deduction instead of the flat $3,350 and a $3,400 exemption instead of $3,200. The net effect overstated taxable income by $500 ($28,034 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It counted the full $73,000 of educational assistance as taxable, getting $107,084 of income, but Maryland AGI is only the $34,083.53 of wages. It also used the repealed $2,700 cap instead of the flat $3,350 deduction. It then misapplied the schedule as well: Maryland's actual brackets tax $101,184 at $4,756.70, not $5,598.54."
-us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It applied an indexed $2,850 standard-deduction cap from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used a $2,000 standard deduction instead of the flat $3,350 and left out the $3,200 personal exemption entirely. That gave $32,084 of taxable income instead of $27,533.53."
-us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a $2,350 'maximum' standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $1,000 ($28,534 instead of $27,533.53)."
-us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $3,100 standard deduction instead of the flat $3,350, getting $1,267.24. It then subtracted an invented $240 'personal exemption credit' and a 'standard deduction credit equivalent', but Maryland has no such nonrefundable credits. Both the exemption and the deduction were already subtracted from income, so this counted them twice."
-us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated the EITC as something that reduces taxable income to $0. The EITC is a credit, not a deduction, and a childless single filer earning $34,084 is above the childless EITC phase-out range anyway. Maryland taxable income is $34,083.53 − $3,350 − $3,200 = $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above $5,250 to income, making AGI $101,834, even though the full $73,000 stays out of Maryland AGI. It then used the old capped standard deduction of $2,700 instead of the flat $3,350 and halved the exemption because of that inflated AGI."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages, though they do not reduce Maryland AGI. Then, even with a $3,900 deduction leaving about $21,800 taxable, it declared the tax zero without applying the $3,200 exemption or the rate schedule. The correct taxable income is $27,533.53, which produces $1,255.34."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Claimed the taxable base was not positive. The $3,350 standard deduction and $3,200 exemption remove only $6,550 from $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Debated including $67,750 of educational assistance in income, worked out $4,554 and $1,336, and then submitted $1,029, which matches neither figure. Both of its paths used the old $2,700 standard deduction cap instead of Maryland's flat $3,350."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of 15% of AGI capped at $2,700 instead of the flat $3,350 Maryland adopted for 2025, which put taxable income at $28,184. It then added a made-up 'personal exemption credit adjustment' to move its own $1,286 up to $1,308."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the old standard deduction cap of about $2,700 instead of the flat $3,350, which overstated taxable income at $28,184. It also reported $1,301, while the rate schedule applied to its own $28,184 gives $1,286.24."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above $5,250 to Maryland income, making AGI $101,834, even though the whole $73,000 stays out of income. It also used a $2,500 standard deduction cap instead of the flat $3,350."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Used a capped standard deduction of about $2,550 instead of the flat $3,350, giving $28,334 of taxable income. It then reported $1,050, below the $1,293 that the 2%–4.75% schedule produces on its own taxable income, so the bracket arithmetic was also wrong."
+us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put the full $73,000 of educational assistance into Maryland AGI, making it $107,084, though none of it belongs there. That inflated AGI triggered an exemption phase-out and the 5% bracket. It also used a $2,400 deduction cap and a $3,300 exemption instead of $3,350 and $3,200."
+us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $93,445 AGI equals wages plus $67,750 of educational assistance minus $8,389 of insurance premiums. That is two mistakes: the educational assistance is not income, and the premiums do not reduce wages. Its $5,980 of combined deductions also falls short of the $6,550 from the flat $3,350 deduction and $3,200 exemption."
+us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance to AGI, making it $101,834, used a $2,400 standard deduction instead of $3,350, and left out the $3,200 personal exemption entirely."
+us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Got the AGI, the $3,200 exemption and the rate schedule right, but used a $2,800 standard deduction instead of Maryland's flat $3,350. That overstated taxable income by about $550, at $28,084 instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages for an AGI of $25,695, though they do not reduce Maryland AGI of $34,083.53. It also used the old standard deduction cap of $2,550 instead of the flat $3,350."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $1,133 implies about $24,960 of taxable income, so it took roughly $9,100 of deductions from $34,084 of wages instead of the correct $6,550 (the $3,350 standard deduction plus the $3,200 exemption), which leaves $27,533.53 taxable."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Subtracted the employer insurance premiums from wages, for an AGI of $25,695, and used a $2,750 standard deduction instead of $3,350. It then added an estimated $592 of county income tax, even though county tax is not part of state income tax before refundable credits."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $98,695 AGI equals wages plus the full $73,000 of educational assistance minus $8,389 of premiums. It should have excluded the educational assistance and left wages alone. It also used a $2,550 standard deduction and a $60 base on the first $3,000 instead of $90."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $997 implies about $22,100 of taxable income, roughly $5,400 below the correct $27,533.53 ($34,083.53 of AGI minus the $3,350 standard deduction and $3,200 exemption), so it took far too much off the income base."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Used a standard deduction 'maximum' of $2,550 from the old 15%-capped formula instead of the flat $3,350 that applies from 2025, which overstated taxable income at $28,334."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $4,786 matches about $101,800 of taxable income taxed partly in the 5% bracket, which only happens if the full $73,000 of educational assistance went into AGI. That assistance stays out of income, and correct taxable income is $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Used a $2,700 standard deduction from the old 15%-of-income cap instead of the flat $3,350, giving $28,184 of taxable income instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Used a $2,750 standard deduction and a $3,300 exemption instead of $3,350 and $3,200. It also misapplied the rates: $90 plus 4.75% of the excess over $3,000 on its own $28,034 is $1,279, not $1,453.28."
+us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used a $3,200 standard deduction instead of Maryland's flat $3,350, which overstated taxable income by $150 at $27,684. It then rounded $1,262.49 to $1,262."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and exemption fully offset the wages. Together they total only $6,550 against $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Floored the tax at zero without ever computing taxable income. The $3,350 standard deduction and $3,200 exemption leave $27,533.53 taxable, and Maryland's 2%–4.75% schedule produces $1,255.34."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used an inflation-adjusted standard deduction of $2,900 instead of Maryland's flat $3,350, which overstated taxable income by $450 at $27,984."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above the $5,250 limit to Maryland AGI, making it $101,834, even though the full $73,000 stays out of income. AGI is just $34,083.53 of wages."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Used a projected standard deduction maximum of $2,850 from the repealed 15%-of-income formula instead of the flat $3,350, giving $28,034 of taxable income instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 taxable income comes from putting the full $73,000 of educational assistance into AGI ($107,084 minus $2,550 minus $3,200). That assistance stays out of income, and it also used $2,550 instead of the flat $3,350 standard deduction."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Used a $2,900 standard deduction instead of Maryland's flat $3,350, which overstated taxable income at $27,984 instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Used a $3,450 standard deduction instead of the flat $3,350. That understated taxable income by $100, at $27,434 instead of $27,533.53, and the tax by $4.75."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave a rough $1,400 without any deductions shown. That figure implies only about $3,500 was taken off wages, when the $3,350 standard deduction plus the $3,200 exemption remove $6,550 and give $1,255.34."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used a $2,350 'maximum' standard deduction from the old 15%-of-income formula instead of the flat $3,350, which overstated taxable income by $1,000 at $28,534."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used a capped standard deduction of about $2,600 instead of Maryland's flat $3,350, giving $28,284 of taxable income instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Used the old 15%-of-AGI rule with a $2,850 cap for the standard deduction, though Maryland switched to a flat $3,350 for single filers starting in 2025. That gave $28,034 of taxable income instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages for an AGI of $25,695, though Maryland AGI is the full $34,083.53. It also used a $2,400 standard deduction instead of $3,350."
+us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of about $2,650 and an exemption of about $3,400 instead of the flat $3,350 and $3,200. That is $6,050 of deductions instead of $6,550, giving $28,034 of taxable income."
+us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Treated the full $73,000 of educational assistance as taxable, making AGI $107,084, though it stays out of income. It also used a made-up 2%–6% rate schedule and a $2,700 standard deduction instead of the flat $3,350."
+us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Used an inflation-adjusted standard deduction cap of $2,850 instead of the flat $3,350 that replaced the 15%-of-income formula in 2025, which overstated taxable income at $28,034."
+us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of about $2,000 instead of $3,350 and left out the $3,200 personal exemption entirely, giving $32,084 of taxable income instead of $27,533.53."
+us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Used a $2,350 'maximum' single standard deduction from the old 15%-capped formula instead of the flat $3,350, which overstated taxable income by $1,000 at $28,534."
+us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Worked out $1,267.24 after deducting the $3,200 exemption and a $3,100 standard deduction (instead of $3,350). It then subtracted a made-up $240 'personal exemption credit' and a 'standard deduction credit equivalent', counting deductions Maryland allows only once a second time."
+us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and EITC bring taxable income to zero. EITC is a credit, not a deduction, and it is zero here, and the $3,350 standard deduction plus the $3,200 exemption leave $27,533.53 taxable."
us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the childless federal EITC phases out to zero at this income but then invented a $245 residual Maryland refundable credit. Applying Maryland’s percentage to a federal EITC of zero yields $0, not a minimal positive amount."
us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated $34,084 as qualifying low income without applying the federal childless EITC phaseout ceiling, then assigned a Maryland EITC amount exceeding the credit pathway it cited. With no qualifying children, federal EITC is zero at this income, so the percentage-based Maryland refundable EITC and the total state refundable credits are both $0."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning gets every step right: $2,172 of tax on $20,170 taxable income, less a 10% Saver's Credit of $192 on $1,924, which gives about $1,980. It then submitted $1,697, which contradicts its own derivation."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the outdated $14,600 single standard deduction instead of the 2026 amount of $16,100. It then guessed at the bracket tax, arriving at about $2,180. It never applied the nonrefundable 10% Saver's Credit of $192.40 on $1,924 of 401(k)/IRA contributions (AGI $36,270)."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly computed $2,172 of tax on $20,170 taxable income, then replaced it with $2,398 under an unexplained 'bracket estimate adjustment'. It also never subtracted the 10% Saver's Credit of $192.40, which AGI of $36,270 qualifies for."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It computed $1,240 + $932 but reported the sum as $2,549 instead of $2,172, an arithmetic error. It also stated that no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40 on $1,924 of retirement contributions."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $16,900 standard deduction instead of $16,100 and computed about $2,076. It then submitted an unrelated $2,544 without deriving it. It also left out the 10% Saver's Credit of $192.40, even though AGI of $36,270 falls in the 10% tier."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on a projected $15,700 standard deduction instead of the enacted 2026 figure of $16,100, which inflated taxable income to $20,570 and tax to $2,220. It also declared 'no nonrefundable credits', missing the $192.40 Saver's Credit (10% of $1,924 in 401(k)/IRA contributions, Roth included)."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly reached $2,172 of pre-credit tax and noted the Saver's Credit, but its answer of $1,329 implies an $843 credit. AGI of $36,270 puts a single filer in the 10% tier, so the credit is $192.40 on $1,924. The $843 reduction matches no Saver's Credit rate."
-us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used a $15,600 standard deduction and a $12,300 top for the 10% bracket. The 2026 figures are $16,100 and $12,400. That overstated taxable income at $20,670 and pre-credit tax at $2,234.40, when the correct figures are $20,170 and $2,172.36."
-us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages even though they are not pre-tax, and it used a $15,360 standard deduction with a $12,211 top for the 10% bracket instead of $16,100 and $12,400. It also computed the Saver's Credit on only the $1,554 of traditional contributions, leaving out the $370 of Roth 401(k)/IRA contributions from the $1,924 base."
-us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It got AGI ($36,270) and the $192.40 Saver's Credit right, but used a $15,450 standard deduction instead of the 2026 amount of $16,100. That inflated taxable income to $20,820 and tax to $2,252.90."
-us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $192.40 Saver's Credit, but ended the 10% bracket at $12,200 instead of the 2026 threshold of $12,400. That added $4 to pre-credit tax ($2,176.40) and produced $1,984 instead of $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026. It applied a $7,600 standard deduction plus a $5,150 personal exemption and a 15% second bracket, so tax before credits came out at $2,948. Under 2026 law the TCJA rates are permanent, with a $16,100 standard deduction, no personal exemption and a 12% bracket, so tax before credits is $2,172.36."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of AGI (using $36,546 instead of $36,270) and used a $15,500 standard deduction instead of $16,100. Its $1,494 answer does not follow from 10%/12% bracket tax on its own $21,046 of taxable income (about $2,270), and it never shows the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got AGI right at $36,270, but subtracted a personal exemption, which follows pre-TCJA sunset law. Its $2,576 answer is consistent with the reverted standard deduction, exemption and 15% bracket, rather than the permanent 2026 parameters: a $16,100 deduction, no exemption, and 10%/12% brackets."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a reverted post-TCJA-sunset standard deduction and personal exemption, which yield $22,370 of taxable income taxed in a 15% bracket for $2,775.50. The 2026 rules are a $16,100 deduction and 10%/12% brackets. It also wrongly subtracted $300 of health insurance premiums from AGI."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It left out the $69 traditional IRA and $276 student loan interest deductions ($36,615 AGI), and used the 2025 $15,000 standard deduction and 2024 $11,600 top for the 10% bracket. It also never applied the $192.40 Saver's Credit, ending at $2,362."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used a $15,700 standard deduction and the 2025 $11,925 top for the 10% bracket. The 2026 figures are $16,100 and $12,400. That overstated pre-credit tax at $2,229.90 instead of $2,172.36."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a personal exemption and reverted standard deduction on the assumption that the TCJA sunset in 2026. The implied pre-credit tax of $2,780.50 reflects the 15% bracket, while 2026 law keeps the $16,100 deduction and 10%/12% brackets for $2,172.36 before the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA expired. It used an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, producing $2,785.50 of pre-credit tax. The 2026 rules are a $16,100 standard deduction, no exemption and a 12% bracket, which give $2,172.36."
-us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used an estimated $15,420 standard deduction and a $12,259 top for the 10% bracket instead of the 2026 figures of $16,100 and $12,400. That inflated tax to $2,256.82."
-us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It correctly computed $2,172 of tax, but concluded no nonrefundable credits apply. At AGI $36,270 the single filer qualifies for the 10% Saver's Credit on $1,924 of 401(k)/IRA contributions, which is $192.40."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits wipe out the regular tax while admitting no such credits are present. In fact $20,169.67 of taxable income produces $2,172.36 of tax, and the only offsetting credit is the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed nonrefundable credits reduce the tax to zero without naming any. The household has no dependents, so the only nonrefundable credit is the $192.40 Saver's Credit against $2,172.36 of tax."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the projected post-TCJA-sunset standard deduction and personal exemption with a 15% bracket ($22,670 taxable income), instead of 2026 law with a $16,100 deduction and 10%/12% brackets. It also said no nonrefundable credits apply, missing the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It computed the pre-credit tax of $2,172.40 correctly, but stated that no nonrefundable credits are available. It missed the 10% Saver's Credit of $192.40 on $1,924 of traditional and Roth retirement contributions at AGI $36,270."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It computed $2,172.40 as a 'no-credit' tax and stopped there. It never subtracted the nonrefundable Saver's Credit of $192.40 (10% of $1,924 in 401(k)/IRA contributions) that AGI of $36,270 qualifies for."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It computed the $2,172.40 bracket tax correctly, but asserted that no nonrefundable credits are used. That missed the $192.40 Saver's Credit, which is 10% of the $1,924 of traditional and Roth retirement contributions."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave no computation. The correct figure is $2,172.36 of 10%/12% tax on $20,169.67 of taxable income, less a $192.40 Saver's Credit. Its $1,623 answer sits $549 below the pre-credit tax, a reduction that no Saver's Credit rate on the $1,924 base produces (10% = $192.40, 20% = $384.80, 50% = $962), so it understated tax with an unsupported deduction or credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset rules': an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, giving $2,806 of tax. 2026 law keeps the $16,100 standard deduction and 10%/12% brackets, which give $2,172.36 before the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed tax under post-TCJA-sunset rules, with a standard deduction plus personal exemption and 10%/15% brackets, giving about $2,810. It should have used the 2026 $16,100 standard deduction and 12% bracket, which give $2,172.36 before the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $192 Saver's Credit, but its $2,176 ordinary tax implies a 10% bracket ending at $12,200 instead of the 2026 threshold of $12,400. That $4 overstatement produced $1,984 instead of $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted health insurance premiums from AGI as if they were pre-tax (AGI $35,970) and used a $15,400 standard deduction instead of $16,100. It also computed a $155 Saver's Credit on only the traditional contributions, leaving the $370 of Roth 401(k)/IRA contributions out of the $1,924 base."
-us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed the childless head as head of household, using a $21,900 standard deduction and HoH brackets. With no qualifying person the head files single, with a $16,100 deduction. It also based the Saver's Credit on only the $1,554 of traditional contributions, leaving out the Roth 401(k)/IRA amounts that make up the $1,924 base."
-us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"By its own figures it reached about $2,108 of tax less a $159 Saver's Credit, yet it submitted $0 on the claim that unnamed 'other' nonrefundable credits eliminate the tax. It also left the $100 of interest and the $276 student loan interest deduction out of taxable income, and used a $15,750 standard deduction instead of $16,100."
-us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It computed the $2,172.40 pre-credit tax and the $1,924 contribution base correctly, but applied the 50% Saver's Credit rate, claiming AGI was below the lowest threshold. At AGI $36,270 a single filer is in the 10% tier, so the credit is $192.40, not $962."
-us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied a 50% Saver's Credit rate to only the traditional contributions ($777), when AGI of about $36,270 puts a single filer in the 10% tier on the full $1,924 base including Roth amounts. It also left the $276 student loan interest deduction out of AGI and used a $15,750 standard deduction and a $12,150 bracket. It then submitted $1,213.50, which contradicts its own $1,475.52."
-us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It overstated pre-credit tax at $3,537 and then zeroed it out with the Credit for Other Dependents, even though the single head has no dependents. The only nonrefundable credit is the $192.40 Saver's Credit against $2,172.36 of tax."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning correctly reaches taxable income of $20,170, tax of $2,172, and a 10% Saver's Credit of $192, for ≈$1,980. It then submitted 1697, which contradicts its own derivation. This is a transcription error, not a rule error."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100. It also subtracted student loan interest after the standard deduction instead of including it in AGI. It never applied the 10% Saver's Credit ($192.40 on $1,924 of contributions), so it landed near the pre-credit tax at $2,180."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It misstated AGI as roughly $35,000 (correct: $36,270) and taxable income as about $18,000 (correct: $20,170). It also never subtracted the nonrefundable 10% Saver's Credit of $192.40, so its $2,010 matches neither the $2,172 pre-credit tax nor the $1,980 post-credit figure."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It correctly computed $2,172 of tax on $20,170 of taxable income, then raised it to $2,398 by an arbitrary 'bracket estimate adjustment.' It also left out the 10% Saver's Credit of $192.40 on $1,924 of retirement contributions."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It computed $1,240 + $932 but stated the total as $2,549 instead of $2,172, an arithmetic error. It also said no nonrefundable credits apply, which misses the Saver's Credit: single-filer AGI of $36,270 qualifies at the 10% rate on $1,924 of contributions."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It used a $16,900 standard deduction instead of the 2026 single amount of $16,100. It then replaced its own ≈$2,076 result with an unsupported $2,544. It also stated that no nonrefundable credits apply, omitting the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used a projected $15,700 standard deduction instead of the OBBBA-set 2026 single amount of $16,100, which overstated taxable income by $400. It also concluded that no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40 on the $1,924 of 401(k) and IRA contributions."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly reached $2,172 of tax before credits. It then subtracted an unexplained ≈$843 to reach $1,329. At AGI $36,270 the Saver's Credit rate is 10%, so the credit is 10% of $1,924 = $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The Saver's Credit is correct at $192.40. However, it used a $15,600 standard deduction and a $12,300 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400. That overstated pre-credit tax at $2,234 versus $2,172."
+us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages even though they are not pre-tax payroll deductions here. It also used a $15,360 standard deduction instead of $16,100. Finally, it based the Saver's Credit only on traditional contributions ($1,554), excluding the Roth 401(k) and Roth IRA amounts that bring the eligible base to $1,924."
+us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"AGI and the $192.40 Saver's Credit are correct. It used a $15,450 standard deduction instead of the 2026 single amount of $16,100, which inflated taxable income to $20,820 and pre-credit tax to $2,252.90."
+us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $12,200 ceiling for the 10% bracket instead of the 2026 single threshold of $12,400. That gives pre-credit tax of $2,176.40 instead of $2,172.36, leaving it $4 high after the correct $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,600 standard deduction, a $5,150 personal exemption, and a 15% bracket. The OBBBA made TCJA rates permanent, so 2026 has a $16,100 single standard deduction, no personal exemption, and a 12% second bracket."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of AGI and used a $15,500 standard deduction instead of $16,100. Its $1,494 does not follow from any stated bracket computation. The correct path is $2,172.36 of tax less the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted a 'personal exemption' along with the standard deduction, which assumes the TCJA sunset. Under OBBBA, 2026 has no personal exemption, a $16,100 standard deduction, and 10%/12% brackets, so its tax is overstated by about $600."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a 'reverted' pre-TCJA standard deduction plus personal exemption and the 15% bracket. OBBBA made the TCJA structure permanent for 2026. It also wrongly subtracted health premiums when computing AGI ($35,970 instead of $36,270)."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It left out the $69 traditional IRA and $276 student loan interest deductions and used a $15,000 standard deduction and a $11,600 10% bracket instead of the 2026 values of $16,100 and $12,400. It also never applied the 10% Saver's Credit of $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The Saver's Credit is correct at $192.40. It used a $15,700 standard deduction and the 2025 $11,925 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400, which overstated pre-credit tax at $2,229.90."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a personal exemption under '2026 law,' which assumes the TCJA sunset. The OBBBA kept the $16,100 standard deduction, zero personal exemption, and 12% second bracket. Pre-credit tax should be $2,172.36, not about $2,780."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The OBBBA made the TCJA rates permanent, so 2026 uses a $16,100 standard deduction with 10%/12% brackets, giving $2,172.36 before the Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"AGI and the $192.40 Saver's Credit are correct. It used an estimated $15,420 standard deduction and a $12,259 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400, which overstated pre-credit tax at $2,256.82."
+us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly computed $2,172 of pre-credit tax but said no nonrefundable credits apply, considering only child-related credits and the EITC. The Saver's Credit applies at the 10% rate for single AGI of $36,270 on $1,924 of contributions, which reduces tax by $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits fully offset the tax, even though the household has no children or dependent care. The only applicable credit is the $192.40 Saver's Credit, which leaves $1,979.96 on $2,172.36 of tax."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed nonrefundable credits reduce tax to zero without naming any qualifying credit. On $20,170 of taxable income the tax is $2,172.36, and the only available nonrefundable credit is the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a 'post-TCJA-sunset' standard deduction plus personal exemption and the 15% bracket, but the OBBBA made the TCJA rates permanent for 2026. It also said no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It correctly reached $2,172.40 of pre-credit tax but asserted that no nonrefundable credits are available. It missed the Saver's Credit: single AGI of $36,270 falls in the 10% tier, so the credit on $1,924 of 401(k) and IRA contributions is $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It computed the no-credit tax of $2,172.40 and stopped there. It never applied the nonrefundable Saver's Credit of 10% on $1,924 of traditional and Roth 401(k)/IRA contributions ($192.40), which applies at single AGI of $36,270."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It reached the correct $2,172.40 of pre-credit tax but said no nonrefundable credits were used. That leaves out the $192.40 Saver's Credit (10% tier at AGI $36,270, on $1,924 of contributions)."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave $1,623 with no computation. The correct taxable income of $20,170 yields $2,172.36 of tax, and the $192.40 Saver's Credit leaves $1,979.96. Its figure understates taxable income or overstates credits by about $357."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset rules' with an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The OBBBA made the TCJA structure permanent, so the 2026 standard deduction is $16,100 with no exemption and a 12% second bracket. Its Saver's Credit of $192 is correct."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used post-TCJA-sunset rules (personal exemption, 10%/15% brackets) to reach $22,670 of taxable income. Under OBBBA the 2026 single standard deduction is $16,100 with no exemption, giving $20,170 taxed at 10%/12%. Its $192 Saver's Credit is correct."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $2,176 of ordinary tax reflects a $12,200 10%-bracket ceiling instead of the 2026 single threshold of $12,400, which gives $2,172.36. That leaves its answer $4 high after the correct $192 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted health insurance premiums from wages (AGI $35,970 instead of $36,270) and used a $15,400 standard deduction instead of $16,100. It also limited the Saver's Credit to traditional contributions ($155), omitting the Roth 401(k) and Roth IRA amounts that make the credit $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed as head of household even though the head has no qualifying person, so the status is single. It therefore used a $21,900 standard deduction and wider HoH brackets instead of the single $16,100 and $12,400. It also computed the Saver's Credit only on traditional contributions ($1,554) instead of all $1,924."
+us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It estimated about $2,108 of tax and a $159 Saver's Credit, then submitted 0 based on unspecified 'other nonrefundable credits' that this childless single filer does not have. The only credit is the $192.40 Saver's Credit, which leaves $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% Saver's Credit rate, claiming AGI was below the lowest single threshold. Single AGI of $36,270 is far above the 50%-tier ceiling and falls in the 10% tier, so the credit is $192.40, not $962."
+us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It left the $276 student loan interest deduction out of AGI and used a $15,750 standard deduction instead of $16,100. It applied a 50% Saver's Credit rate on traditional contributions only, when AGI of $36,270 qualifies for just 10% of the full $1,924. It then submitted $1,213.50, which contradicts its own $1,475.52."
+us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed the Credit for Other Dependents eliminates the tax, but the household has no dependents, so that credit is $0. It also overstated pre-credit tax at $3,537 instead of $2,172.36. The only applicable credit is the $192.40 Saver's Credit."
us,scenario_070,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied the Additional Child Tax Credit earned-income formula despite the household having no qualifying child, so there was no Child Tax Credit balance available to refund. Its separate EITC rationale also misstates the rule: $100 of investment income does not exceed the limit; the childless EITC is zero because $38,000 of earnings exceeds its phaseout ceiling."
us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model called eligibility borderline without calculating the stated household’s MAGI-to-FPL ratio. At 2.27 times FPL, the head exceeds Illinois’s 138% FPL ACA expansion limit, and age 57, low assets, and employer-sponsored insurance do not establish another Medicaid category."
us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $300 health insurance premium as a pre-tax FICA exclusion even though no pre-tax treatment was specified. FICA applies to the full $38,000, yielding $2,356 of Social Security tax and $551 of Medicare tax."
@@ -5716,41 +5869,42 @@ us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"T
us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax."
us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907."
us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Used the correct $36,270 AGI base but subtracted the 2025 personal exemption of $2,850 instead of the 2026 indexed exemption of $2,925. That left taxable income of $33,420 instead of $33,345 and tax of $1,654.29 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Never excluded the $1,485 traditional 401(k) deferral from wages, so its base was $37,755 instead of $36,270. It also treated the exemption as a $120 credit based on an outdated $2,425 figure instead of deducting the $2,925 exemption from income. Its $1,154 answer does not even follow from its own $1,869 minus $120."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly derived $36,270 AGI but subtracted the 2025 exemption of $2,850 instead of the 2026 exemption of $2,925. It then inflated its own $1,654.29 result to $1,659.83 with an unexplained rounding adjustment, moving further from $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Computed the correct $36,270 AGI but used the 2024 exemption of $2,775, which gives $1,658, instead of $2,925. It then abandoned that figure for an unexplained taxable income of $34,870 (implying an exemption of only $1,400) and reported $1,726."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Used the $36,270 AGI with the 2025 exemption of about $2,850 instead of the 2026 exemption of $2,925, which gave about $1,655. It then raised that to $1,699 through unexplained base and rounding adjustments that have no basis in Illinois law."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Wrongly added the $69 IRA deduction and $276 student loan interest back to Illinois base income; Illinois base income starts from federal AGI and has no addition modification for either. It also subtracted a $300 health premium from wages that the case does not treat as a pre-tax exclusion. Finally it used the outdated $2,425 exemption instead of $2,925, which produced $33,890 of taxable income instead of $33,345."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of federal AGI, giving $36,546 instead of $36,270; Illinois inherits that deduction through AGI. It also used the 2025 exemption of $2,850 instead of $2,925, then added an arbitrary upward adjustment from $1,668 to $1,690."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Applied 4.95% to $36,270 AGI less the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925. That overstated taxable income by $75 and gave about $1,654 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Invented a $2,450 Illinois standard deduction on top of a $2,450 personal exemption. Illinois has no standard deduction, only a single personal exemption, which is $2,925 for 2026. It therefore subtracted $4,900 instead of $2,925 and understated the tax at $1,552.82."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $35,670 by deducting the $600 of listed health insurance premiums, which do not reduce federal AGI here; the correct figure is $36,270. It also used a $2,842 exemption instead of the 2026 exemption of $2,925, which gave $32,828 of taxable income and $1,624.99 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Subtracted the 2025 personal exemption of $2,850 from $36,270 AGI instead of the 2026 exemption of $2,925. That gave taxable income of $33,420 and tax of $1,654.29 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Applied a nonrefundable $135 renter's credit, treating 25% of rent as property tax, but Illinois has no such credit; its property tax credit requires property tax paid on a residence, and this household is a renter with no property tax. It also used the 2024 exemption of $2,775 instead of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Cited a $2,775 exemption instead of the 2026 exemption of $2,925. Its $1,716 answer implies about $34,667 of taxable income, or roughly $37,442 of base income before the exemption, so it also failed to reduce income to the $36,270 federal AGI."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Started from the correct $36,270 AGI but subtracted an outdated exemption of about $2,500 instead of the 2026 indexed exemption of $2,925. That overstated taxable income by about $425 and gave $1,671."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,970, subtracting a $300 health premium that does not reduce AGI here; the correct figure is $36,270. It also used the 2024 exemption of $2,775 instead of $2,925, which gave $33,195 of taxable income and $1,643.15 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no figures, but its $1,683 answer implies $34,000 of taxable income. Measured from the $36,270 AGI, that is an exemption of only about $2,270 instead of the 2026 exemption of $2,925, which overstates the base by about $655."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 from $36,270 AGI instead of the 2026 indexed exemption of $2,925. That gave $33,420 of taxable income and $1,654.29 of tax instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used the 2024 personal exemption of $2,775 instead of the 2026 exemption of $2,925. That overstated taxable income by $150 and gave $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Subtracted a $2,775 exemption, the 2024 value, from $36,270 AGI instead of the 2026 exemption of $2,925. That gave $33,495 of taxable income and $1,658 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Omitted the $276 student loan interest deduction from federal AGI, giving $36,546 instead of $36,270. It also used the outdated $2,425 exemption instead of $2,925, and then submitted $1,626, which contradicts its own computed $1,689."
-us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Applied 4.95% to $36,270 AGI less the 2025 exemption of about $2,850 instead of the 2026 indexed exemption of $2,925. That gave about $1,654 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the exemption and retirement deductions reduce Illinois tax to zero. In fact the only reductions are the 401(k), IRA and student loan items already reflected in the $36,270 AGI, plus one $2,925 exemption, which leaves $33,345 taxable at 4.95% for $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that retirement contributions and student loan interest drive Illinois liability to zero. Those adjustments only reduce AGI to $36,270, and after the $2,925 exemption $33,345 remains taxable at the 4.95% flat rate."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of AGI, giving $36,546 instead of $36,270. It also used a $1,824 exemption, far below the 2026 Illinois exemption of $2,925, which gave $34,722 of taxable income and $1,718.72 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used the 2025 personal exemption of $2,850 instead of the 2026 exemption of $2,925. That gave $33,420 of base income and $1,654.29 of tax instead of $33,345 and $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no figures, but its $1,777 answer implies about $35,899 of taxable income, about $2,555 above the correct $33,345 ($36,270 AGI less the $2,925 exemption). It therefore failed to reduce wages to federal AGI and to apply the full 2026 exemption."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Correctly started from $36,270 AGI but subtracted the 2024 exemption of $2,775 instead of the 2026 exemption of $2,925. That gave $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,970, deducting a $300 health premium that does not reduce AGI here, instead of $36,270. It also used the outdated $2,425 exemption instead of $2,925, which gave $33,545 of taxable income and about $1,660 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of about $2,775 from $36,270 AGI instead of the 2026 indexed exemption of $2,925. That gave $33,495 of taxable income and $1,658 of tax."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Stated that Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a personal exemption of $2,925 for 2026, so taxable income is $33,345 and tax is $1,650.56, not $1,795."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly found no additions or subtractions and used $36,270 as base income, but subtracted the 2024 exemption of $2,775 instead of the 2026 exemption of $2,925. That gave $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois standard deduction that cut taxable income to about $20,696, then reported $1,106 without a coherent base. Illinois taxable income is federal AGI of $36,270 less only the $2,925 exemption, which gives $33,345."
-us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used the correct $36,270 AGI but approximated the 2026 exemption as $2,900 instead of the indexed $2,925. That left taxable income $25 too high and gave $1,651.82 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Subtracted a $2,800 exemption from income, then applied the same $2,800 again as a dollar-for-dollar nonrefundable credit against tax, which zeroed out the liability. The Illinois exemption is only a $2,925 deduction from base income, not a credit, so tax is 4.95% of $33,345, or $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $1,658.66 answer implies about $33,508 of taxable income. That corresponds to an exemption of about $2,762, near the 2024 value of $2,775, subtracted from $36,270 AGI instead of the 2026 exemption of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Used the 2025 Illinois personal exemption of $2,850 instead of the 2026 amount of $2,925. That leaves taxable income at $33,420 instead of $33,344.67, so the tax comes to $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Left the $1,485 traditional 401(k) deferral in the base, which Illinois picks up from federal AGI. Treated the exemption as a $120 credit instead of a $2,925 deduction from base income. Its own figures ($1,869 − $120) don't add up to the $1,154 it submitted; the correct result is 4.95% × $33,344.67 = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois 'retirement subtraction' that cut the base to about $20,000. Illinois base income is federal AGI ($36,270) minus only the $2,925 exemption, which gives $33,344.67 and $1,650.56 of tax."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used the outdated $2,850 exemption instead of the 2026 amount of $2,925. It then raised its own $1,654.29 result to $1,659.83 with no basis, when the correct figure is $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. It then dropped its $1,658 result and substituted an unsupported taxable income of $34,870. The correct taxable income is $36,270 − $2,925 = $33,344.67, giving $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Used the $2,850 exemption instead of the 2026 amount of $2,925, then raised its own ~$1,655 figure to $1,699 for 'rounding' with no basis. The correct liability is 4.95% × $33,344.67 = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Added back the IRA deduction and student loan interest, but Illinois has no such add-backs because base income is federal AGI. Subtracted the $300 health premium as a pre-tax wage reduction even though the facts don't say it is pre-tax. Used the 2023 exemption of $2,425 instead of $2,925, producing $33,890 of taxable income instead of $33,344.67."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Left out the $276 student loan interest deduction on the claim that Illinois disallows it, but the deduction is already built into federal AGI, which is where Illinois starts. Used the $2,850 exemption instead of $2,925, then added an unsupported 'phase-in' adjustment to reach $1,690."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $33,420 of taxable income and $1,654 of tax instead of $33,344.67 and $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Invented an Illinois standard deduction of $2,450 and stacked it on a $2,450 exemption. Illinois allows only the personal exemption, which is $2,925 for 2026, so taxable income is $33,344.67, not $31,370."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Understated AGI by $600 ($35,670 instead of about $36,270), which matches subtracting both $300 health premium entries, neither of which reduces wages. Also used a $2,842 exemption instead of the 2026 amount of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654.29 instead of 4.95% × $33,344.67 = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the 2024 exemption of $2,775 instead of $2,925. It also invented a $135 nonrefundable renter's credit, but Illinois's property tax credit applies only to property tax paid on a principal residence and is not available to renters on rent."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of $2,925. Its $1,716 answer implies about $34,667 of taxable income, which means it started from an inflated base of roughly $37,440 instead of the $36,270 federal AGI."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Subtracted a rough $2,500 exemption instead of the 2026 amount of $2,925. That leaves taxable income at about $33,770 instead of $33,344.67 and tax at $1,671 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Took $300 off AGI ($35,970 instead of about $36,270) by treating a health insurance premium as a pre-tax wage reduction, which the facts don't support. Also used the 2024 exemption of $2,775 instead of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $1,683 implies $34,000 of taxable income, which means it took only about $2,270 off the $36,270 AGI. The 2026 exemption is $2,925, giving $33,344.67 of taxable income and $1,650.56 of tax."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $33,420 of net income and $1,654.29 of tax instead of $33,344.67 and $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That overstates taxable income by $150 and gives $1,658 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That leaves taxable income at $33,495 instead of $33,344.67 and tax at $1,658 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270) and used the 2023 exemption of $2,425 instead of $2,925. It also submitted $1,626, which doesn't match its own $1,689 calculation."
+us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654 instead of 4.95% × $33,344.67 = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the exemption and retirement deductions wipe out Illinois tax. Federal AGI of $36,270 minus the single $2,925 exemption still leaves $33,344.67 taxable at 4.95%, so the liability is $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Concluded the retirement and student loan adjustments bring Illinois liability to zero. After those adjustments, AGI is still $36,270 and only a $2,925 exemption comes off, leaving $33,344.67 taxed at 4.95% = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270). It also used an exemption of $1,824 instead of the 2026 amount of $2,925, which overstates taxable income at $34,722."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $1,777 implies about $35,900 of taxable income, so it took only about $370 off the $36,270 AGI instead of the full $2,925 exemption. The correct taxable income is $33,344.67, giving $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That gives $1,658 instead of 4.95% × $33,344.67 = $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Understated AGI as $35,970 by subtracting a $300 health premium as though it were a pre-tax wage reduction, and used the 2023 exemption of $2,425 instead of $2,925. The two errors roughly cancel, but the result is still $1,660 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used the 2024 exemption of about $2,775 instead of the 2026 amount of $2,925. That leaves taxable income at $33,495 instead of $33,344.67 and tax at $1,658 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Stated that Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a $2,925 personal exemption in 2026, which brings taxable income down to $33,344.67 and tax to $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That gives $33,495 of net income and $1,658 of tax instead of $33,344.67 and $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois standard deduction that cut taxable income to about $20,696, then made an unsupported upward adjustment to $1,106. Illinois taxes federal AGI less only the $2,925 exemption, which is $33,344.67 here."
+us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Estimated the 2026 exemption at $2,900 instead of the actual $2,925. That leaves taxable income at $33,370 instead of $33,344.67 and tax at $1,651.82 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Counted the exemption twice: once as a deduction from income and again as a $2,800 dollar-for-dollar nonrefundable credit, which zeroed out the tax. Illinois has only the exemption deduction ($2,925 in 2026), so the liability is $1,650.56. It also left the $276 student loan interest out of AGI."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $1,658.66 implies about $33,508 of taxable income, which matches subtracting the 2024 exemption of about $2,775 instead of the 2026 amount of $2,925. The correct taxable income is $33,344.67, giving $1,650.56."
us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0."
us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE."
us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE."
@@ -5813,49 +5967,50 @@ us,scenario_073,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability
us,scenario_073,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"It assumed the 24-month SSDI waiting period was met for an ongoing disability, but the months of receipt are unlisted and count as 0. It also misread the input name 'health insurance premiums excluding Medicare Part B' as evidence of Medicare enrollment. That label only names the premium variable and says nothing about Part B coverage."
us,scenario_073,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"It said SSDI receipt gives Medicare eligibility under PolicyEngine rules and never applied the 24-month waiting period. With the months of receipt at their default of 0 and the head aged 57, the head is not eligible."
us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no payroll-taxable wages and explicitly derived payroll tax of $0, but then submitted $2,846. Its numeric output contradicts its own payroll-tax-base reasoning and final stated derivation."
-us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model acknowledged Michigan's BBCE 200% FPL gross test and computed net income of about $1,757, which is close to the correct $1,755. It then still applied the 100% FPL net income test, even though categorical eligibility waives it. Because it treated the household as ineligible, it never applied the $24 monthly minimum allotment for eligible one-person households."
-us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model routed the household through the elderly/disabled path, which skips only the gross test, and failed it on the 100% FPL net income limit using net income of $1,747. It missed that Michigan's TANF non-cash categorical eligibility waives the net income test entirely, so the eligible household receives the $24 monthly minimum allotment."
-us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the standard 130% FPL gross income limit and a $2,750 asset limit and declared the household ineligible. Michigan's BBCE applies a 200% FPL gross limit, which the household passes at 165% of FPL, and categorical eligibility removes the asset limit, so the $3,600 in assets does not disqualify it. The eligible household receives the $24 monthly minimum, not $0."
-us,scenario_073,snap,claude-opus-4.7,llm_error,period_annualization,False,"The model subtracted an annual medical expense figure (about $5,165) from monthly income. That drove net income to about zero and gave the full $298 maximum for all 12 months. With monthly figures, net income is $1,755, and the 30% contribution of $527 exceeds the $298 maximum. The benefit therefore falls to the $24 minimum, or $288 a year."
-us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model compared income to the 130% FPL gross limit and then to the 100% FPL net income limit, and concluded the household was ineligible. It missed that Michigan's BBCE uses a 200% FPL gross limit and waives the net income test. The categorically eligible one-person household therefore receives the $24 monthly minimum allotment."
-us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used a one-person maximum allotment of about $587 a month instead of the FY2026 figure of $298. It also counted the $5,789 employer-sponsored insurance premiums as out-of-pocket medical costs, which understated net income at $1,242. Against the correct $298 maximum, 30% of net income wipes out the formula benefit, so only the $24 minimum applies, not $214 a month."
-us,scenario_073,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model correctly computed net income of about $1,747 but treated the 100% FPL net income limit as binding for this disabled household and zeroed the benefit. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household stays eligible and receives the $24 monthly minimum allotment."
-us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test: $1,752 against a limit it put at $1,255. From that, it declared the household ineligible. It missed that Michigan's BBCE confers categorical eligibility, which removes the net income test. The eligible one-person household therefore gets the $24 minimum allotment instead of $0."
-us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model found that the 30% net-income contribution fully offsets the maximum benefit and also cited a $1,255 net income limit, then set SNAP to $0. It skipped the rule that eligible one- and two-person households receive the minimum allotment, $24 a month in FY2026, whenever the formula benefit is below it. It also missed that categorical eligibility waives the net income limit."
-us,scenario_073,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model computed net income of about $1,750 and denied benefits because that exceeds the 100% FPL net limit for elderly/disabled households. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household remains eligible and receives the $24 monthly minimum allotment."
-us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly saw that the 30% contribution exceeds the maximum allotment and that the minimum benefit applies. However, it used about $25 a month instead of the FY2026 minimum of $24, which is 8% of the $298 one-person maximum, rounded. That gave $300 a year instead of $288."
-us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the 130% FPL gross income limit and found the household ineligible. It missed that Michigan's BBCE raises the gross limit to 200% FPL, which the household passes at 165%, and that SSDI receipt exempts this disabled household from the standard gross test. The eligible household receives the $24 monthly minimum allotment."
-us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model denied SNAP because net income exceeds the one-person net income limit. Categorical eligibility through Michigan's TANF non-cash rules waives the net income test, so the household stays eligible and receives the $24 monthly minimum allotment rather than $0."
-us,scenario_073,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model said gross and net income exceed SNAP limits and returned $0. The household passes Michigan's 200% FPL BBCE gross limit, and categorical eligibility waives the net income test, so it is eligible and receives the $24 monthly minimum allotment, or $288 a year."
-us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly found BBCE eligibility and applied the one-person minimum benefit, but it used the FY2025 minimum of $23 instead of the FY2026 minimum of $24. The FY2026 minimum is 8% of the $298 maximum, rounded, so the annual total is $288 rather than $276."
-us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said countable income exceeds the eligibility threshold and returned $0. Michigan's BBCE 200% FPL gross limit is met at 165% of FPL, and categorical eligibility waives the net income test. The household is therefore eligible, and the $24 monthly minimum allotment applies."
-us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the maximum allotment but then paid $0. It missed that eligible one-person households receive the minimum allotment, $24 a month in FY2026, when the formula benefit falls below it."
-us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model computed a 30% contribution of $462.45, more than the maximum benefit, and stopped at $0. It never applied the minimum allotment that every eligible one- or two-person household receives: $24 a month in FY2026, or $288 a year."
-us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that assets and income exceed SNAP limits without computing them. Michigan's TANF non-cash categorical eligibility waives the asset and net income tests, and the household passes the 200% FPL gross limit at 165%. It is therefore eligible for the $24 monthly minimum allotment."
-us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly identified BBCE categorical eligibility and the minimum-benefit outcome, but it used $276 a year, the outdated $23 monthly minimum. The FY2026 minimum is $24, which is 8% of the $298 one-person maximum, rounded, so the annual total is $288."
-us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model zeroed the benefit because net income is too high for a positive allotment. It missed both that categorical eligibility waives the net income limit and that eligible one-person households receive the $24 monthly minimum allotment regardless of the formula result.
-us,scenario_073,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model applied the net income eligibility threshold and the 30% offset to reach $0. Michigan's BBCE categorical eligibility waives the net income test, and eligible one-person households get the $24 monthly minimum allotment even when 30% of net income exceeds the maximum."
-us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model found the household eligible under the net income test and computed a negative formula benefit, then paid $0. It omitted the minimum allotment for eligible one- and two-person households, $24 a month in FY2026. Its net income was also too low because it counted the employer-sponsored insurance premiums as medical costs, though the minimum applies either way."
-us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model compared total gross income of $25,754 directly to a 100% FPL net income limit, without taking any deductions. Categorical eligibility waives the net income test, and the household's $1,755 monthly net income leaves it eligible for the $24 minimum allotment, or $288 a year."
-us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model called income well above SNAP limits. The household's gross income is 165% of FPL, under Michigan's 200% BBCE limit, and categorical eligibility waives the net income test. The eligible one-person household therefore receives the $24 monthly minimum allotment."
-us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said eligibility was not established and returned $0, even though the prompt instructs it to assume take-up. The household is categorically eligible through Michigan's BBCE: gross income is 165% of FPL, under the 200% limit. The formula benefit is below zero, so the $24 monthly minimum allotment applies."
-us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model said no elderly or disabled status was listed, overlooking that SSDI receipt makes the head disabled for SNAP, and failed the household on the gross income test. The household also passes Michigan's 200% FPL BBCE gross limit at 165% of FPL, which confers categorical eligibility and the $24 monthly minimum allotment."
-us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly found that the 30% benefit reduction wipes out the formula allotment, but then returned $0. It skipped the minimum allotment rule that guarantees eligible one-person households $24 a month in FY2026."
-us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model took the standard and medical deductions, found that the remaining income yields no formula benefit, and returned $0. It missed that this categorically eligible one-person household receives the $24 monthly minimum allotment whenever the formula falls below it."
-us,scenario_073,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model concluded that no positive allotment remains without shelter costs, and returned $0. It omitted the minimum allotment, $24 a month in FY2026, which eligible one- and two-person households receive whenever the formula benefit falls below it."
-us,scenario_073,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model noted about $2,146 of monthly income with no shelter costs and set SNAP to zero. The household is categorically eligible, so even though 30% of its $1,755 net income exceeds the $298 maximum, it receives the $24 monthly minimum allotment, or $288 a year."
-us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model denied SNAP because income exceeds the one-person net income limit. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household remains eligible and receives the $24 monthly minimum allotment."
-us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,The model correctly placed the disabled household under the higher asset limit and found that the 30% reduction fully offsets the maximum allotment. It then paid $0 instead of applying the $24 monthly minimum allotment that eligible one-person households receive.
-us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum allotment and set the benefit to $0. It missed the minimum allotment rule, under which an eligible one-person household receives $24 a month in FY2026 when the formula benefit falls below it."
-us,scenario_073,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model denied SNAP because net income stays above 100% of poverty after the medical and standard deductions. Categorical eligibility through Michigan's BBCE waives the net income test, so the household is eligible and receives the $24 monthly minimum allotment."
-us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model computed that 30% of net income, about $382, exceeds the maximum allotment and returned $0. It skipped the minimum benefit for eligible one- and two-person households, $24 a month in FY2026. It also counted the employer-sponsored insurance premiums as medical expenses, though the minimum applies either way."
-us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"The model correctly applied the one-to-two-person minimum benefit, but it used about $23 a month, the FY2025 figure. The FY2026 minimum is $24, which is 8% of the $298 one-person maximum, rounded, so the annual total is $288 rather than $276."
-us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum and paid no benefit. It omitted the minimum allotment that every eligible one-person household receives, $24 a month in FY2026, or $288 a year."
-us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit to a disabled household and zeroed the benefit because 30% of net income exceeds $298. Categorical eligibility waives the net limit, and the $24 monthly minimum allotment applies. The model also skipped the medical deduction for the $2,600 in health premiums."
-us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model said income exceeds both the gross and net limits. The household's 165% FPL gross ratio passes Michigan's 200% BBCE gross limit, and categorical eligibility waives the net income test. The household is eligible for the $24 monthly minimum allotment."
-us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed that Michigan's BBCE retains a 100% FPL net income test and denied benefits on that basis. BBCE categorical eligibility removes the net income test, so the household is eligible and receives the $24 monthly minimum allotment. The model also wrongly dropped the elderly/disabled medical deduction for this SSDI recipient."
-us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model denied disabled status despite the head's SSDI receipt. It then applied a 130% FPL gross limit computed at only about $1,311 a month and declared the household ineligible. Michigan's BBCE applies a 200% FPL gross limit, which the household passes at 165%, so it is categorically eligible for the $24 monthly minimum allotment."
-us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model ruled the household ineligible because it treated $3,600 in bank and stock assets as over the resource limit. That amount is under the limit for a household with a disabled member. Michigan's TANF non-cash categorical eligibility also removes the asset test, so the household is eligible for the $24 monthly minimum allotment."
+us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly passed the 200% FPL BBCE gross test and then applied the 100% FPL elderly/disabled net income test anyway. Because the household is categorically eligible, that net test is waived. Net income of about $1,757 gives a negative formula benefit, so the $24/month minimum allotment for a one-person household applies, which is $288 a year, not $0."
+us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income ($1,747) is above the 100% FPL net limit. Under Michigan BBCE (TANF non-cash), a categorically eligible household skips the net income test, so the negative formula benefit is raised to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross limit and a $2,750 asset limit. It missed that Michigan's BBCE pathway uses a 200% FPL gross test, which the household passes at about 165% FPL. It also missed that this disabled household's $3,600 in assets passes the asset test. Categorical eligibility combined with a negative formula benefit yields the $24/month minimum, $288 a year."
+us,scenario_073,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model's $0 matches stopping where 30% of net income (about $527) exceeds the $298 maximum allotment. It never applied the minimum allotment that a categorically eligible one-person household receives under Michigan BBCE: $24/month, $288 a year."
+us,scenario_073,snap,claude-opus-4.7,llm_error,period_annualization,False,"The model treated about $5,200 of annual health premiums as a monthly medical deduction, which pushed net income below zero and produced the full $298 maximum. It also counted the pre-tax employer-sponsored premiums. The countable excess medical deduction is only about $182/month, so net income is $1,755. The formula benefit is negative, and the household gets only the $24 minimum ($288 a year)."
+us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invoked the 130% FPL gross limit and the 100% FPL net limit to deny eligibility. Michigan's BBCE pathway uses a 200% FPL gross test, which the household passes, and it waives the net test. The categorically eligible single-person household therefore receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used a $587 maximum allotment, which is not the one-person figure; the one-person maximum is $298. It also put the $5,789 of pre-tax employer premiums into the medical deduction. Even with its understated $1,242 net income, 30% of that ($373) exceeds the $298 maximum, so the benefit falls to the $24/month minimum, $288 a year."
+us,scenario_073,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit to the $1,747 net income and returned $0. A household categorically eligible under Michigan BBCE is exempt from the net test, so its negative formula benefit is raised to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model used the 130% FPL gross test and a 100% FPL net test and found the household ineligible. Michigan BBCE grants categorical eligibility up to 200% FPL gross and does not apply a net income test. The household therefore receives the $24/month one-person minimum allotment, $288 a year."
+us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model compared net income with a $1,255 net limit and concluded the 30% offset zeroes out the benefit. It ignored that a BBCE categorically eligible one-person household receives the minimum allotment ($24/month, $288 a year) whenever the formula benefit falls below it."
+us,scenario_073,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model denied benefits because net income of about $1,750 exceeds the 100% FPL limit. Michigan's TANF non-cash BBCE pathway makes the household categorically eligible without a net test, and the $24/month minimum allotment applies, $288 a year."
+us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly reached the minimum-allotment outcome but used $25/month instead of the 2026 minimum of $24/month (8% of the $298 one-person maximum), which gives $300 instead of $288. It also inflated the medical deduction by counting pre-tax employer-sponsored premiums."
+us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the standard 130% FPL gross income test. It ignored Michigan's 200% FPL BBCE threshold, which the household passes at about 165% FPL, and the disabled-member exemption. The categorically eligible household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model denied benefits under a net income limit. A household categorically eligible through Michigan BBCE is not subject to the net test, and the $24/month one-person minimum allotment applies when 30% of net income exceeds the maximum, which totals $288 a year."
+us,scenario_073,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model said gross and net income exceed SNAP limits. Gross income is about 165% FPL, which is under Michigan's 200% BBCE limit, and the categorically eligible household skips the net test. That yields the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly applied BBCE categorical eligibility and the minimum allotment but used the outdated $23/month minimum. The minimum in effect for 2026 is $24/month, so the annual total is $288, not $276."
+us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said income exceeds the threshold, but gross income of about 165% FPL passes Michigan's 200% FPL BBCE test. The categorically eligible single-person household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the $298 maximum allotment but stopped at $0. A categorically eligible one-person household is guaranteed the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the maximum benefit and returned $0, without applying the $24/month minimum allotment owed to a categorically eligible one-person household ($288 a year). Its $1,541.50 net income also overstated deductions; the countable figure is $1,755."
+us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model claimed assets and income exceed SNAP limits. The $3,600 in assets passes the asset test, and gross income passes Michigan's 200% FPL BBCE test. The categorically eligible household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum benefit but used the outdated $23/month minimum. The 2026 minimum is $24/month, which gives $288 rather than $276."
+us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model applied a net income cutoff. Michigan BBCE categorical eligibility removes the net test, and the household's negative formula benefit is raised to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model applied the net income threshold and the 30% offset to reach $0. It missed that the household is categorically eligible under Michigan BBCE, so the net test does not apply and the $24/month minimum allotment does, $288 a year."
+us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model computed a negative benefit and paid nothing, overlooking the minimum allotment for eligible one-person households: $24/month, $288 a year. It also inflated the medical deduction by counting $5,789 of pre-tax employer premiums; countable net income is $1,755, not $1,053."
+us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model compared annual gross income directly with the 100% FPL net limit, skipping both the deductions and categorical eligibility. Under Michigan BBCE the household passes the 200% FPL gross test, has no net test, and receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that income is above SNAP limits. Gross income of about 165% FPL is under Michigan's 200% FPL BBCE limit, so the household is categorically eligible and receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0 on the grounds that eligibility was not established, even though the prompt says to assume take-up. The household is categorically eligible through Michigan BBCE, and its negative formula benefit is raised to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gpt-5.6-luna,llm_error,age_disability,False,"The model said no disabled status was listed, missing that SSDI receipt makes the head disabled for SNAP. It also applied a standard gross test instead of Michigan's 200% FPL BBCE test. The categorically eligible household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model concluded that the 30% benefit reduction leaves no positive allotment but did not apply the $24/month minimum allotment guaranteed to categorically eligible one- and two-person households, which gives $288 a year."
+us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model stopped once the net income offset exceeded the maximum allotment. The BBCE categorically eligible one-person household receives the $24/month minimum allotment instead of $0, $288 a year."
+us,scenario_073,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model correctly saw that no shelter deduction exists and that the formula benefit is not positive. It omitted the $24/month minimum allotment for a categorically eligible single-person household, which gives $288 a year."
+us,scenario_073,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model computed zero support from roughly $2,146 of monthly income with no shelter costs and never applied the minimum allotment. The household is categorically eligible under Michigan BBCE and receives $24/month, $288 a year."
+us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the net income limit to deny eligibility. Michigan BBCE categorical eligibility waives the net test, and the household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly found that the 30% reduction fully offsets the maximum allotment but left the benefit at zero. A categorically eligible one-person household gets the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum allotment and returned $0, omitting the $24/month minimum allotment for categorically eligible one-person households ($288 a year). It also dropped the excess medical deduction available to this disabled member."
+us,scenario_073,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test. Households categorically eligible through Michigan BBCE are exempt from that test, and the household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model found that 30% of net income exceeds the maximum allotment and returned $0 without applying the $24/month minimum allotment for a categorically eligible one-person household ($288 a year). It also overstated the medical deduction by including pre-tax employer-sponsored premiums.
+us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"The model correctly applied the minimum-allotment rule against the $298 FY2026 maximum but used the outdated $23/month minimum. The FY2026 minimum is $24/month (8% of $298, rounded), which gives $288 instead of $276."
+us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the one-person maximum but returned $0. The categorically eligible single-person household is entitled to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net test, which BBCE categorical eligibility waives. It also omitted the excess medical deduction for the disabled member. With the household categorically eligible, the negative formula benefit is raised to the $24/month minimum, $288 a year."
+us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model claimed the household exceeds both the gross and net limits. Gross income is under Michigan's 200% FPL BBCE threshold, and categorical eligibility removes the net test, so the household receives the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model wrongly applied a 100% FPL net test under BBCE and wrongly omitted the excess medical deduction for the SSDI-disabled member. A categorically eligible household is not subject to the net test, and its negative formula benefit is raised to the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross test with a misstated $1,311 limit. It ignored both the disabled-member gross test exemption and Michigan's 200% FPL BBCE threshold, which the household passes. Categorical eligibility then yields the $24/month minimum allotment, $288 a year."
+us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model treated $3,600 in assets as over the resource limit. A household with a disabled member passes the asset test at that level, and the household is categorically eligible through Michigan BBCE. It therefore receives the $24/month minimum allotment, $288 a year."
us,scenario_074,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_074,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model stated that tax-exempt SSDI counts in MAGI and then computed the opposite, summing only the $384 taxable IRA distribution plus a ""taxable portion"" of Social Security — applying the §86 provisional-income taxability test instead of the §36B(d)(2)(B) MAGI definition, which adds back the entire Social Security benefit excluded from gross income. Counting the full $33,640 SSDI plus the $384 IRA distribution puts the head at 2.13 x FPL, roughly 213% of poverty and far above Louisiana's 138% FPL expansion-adult limit."
@@ -5864,62 +6019,64 @@ us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli
us,scenario_074,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model excluded SSDI from MAGI on the reasoning that it is untaxable, reaching a MAGI of $3,410; Medicaid MAGI explicitly adds back Social Security benefits not included in gross income, so the $33,640 SSDI counts dollar-for-dollar. The correct MAGI of roughly $34,000 equals 2.13 x FPL, about ten times the model's figure and well above the ~$22,000 expansion-adult threshold it cited."
us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model asserted ""very low countable income"" with no derivation, a conclusion consistent only with dropping the entire $33,640 SSDI from countable income as non-taxable. Medicaid MAGI adds back untaxed Social Security benefits in full, yielding 2.13 x FPL — above Louisiana's 138% FPL expansion limit — and with age 54, zero SSI, and no dependents there is no non-MAGI category, so medicaid_category is NONE."
us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model restricted the MAGI base to ""the listed taxable income,"" which omits the untaxed SSDI that §36B(d)(2)(B) requires to be added back to AGI for Medicaid MAGI. Including the $33,640 SSDI with the $384 taxable IRA distribution gives 2.13 x FPL, so the head exceeds the 138% FPL expansion-adult limit and qualifies under no MAGI category."
-us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"Stretched the instruction that disability status is constant for the full year into 24 months of SSDI entitlement. A full tax year covers at most 12 months, and the unlisted SSDI-duration input defaults to 0, so the disability pathway fails and the 54-year-old head fails the age-65 test."
-us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated positive Social Security disability income as enough for Medicare on its own. The disability pathway requires 24 months of SSDI receipt, a separate input that was not listed and is therefore 0, so only the age-65 test applies and the head, at 54, fails it."
-us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Reasoned that current SSDI receipt means the 24-month waiting period has already passed. The prompt forbids inferring unlisted status facts, and the months of SSDI receipt default to 0, so the head qualifies only by age and is ineligible at 54."
-us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Used year-long constant status to assume the 24-month SSDI waiting period was met. A constant status across one tax year supplies at most 12 months, and the unlisted receipt-duration input is 0, so the under-65 disability pathway fails and the head is not Medicare eligible."
-us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Correctly named the 24-months-of-SSDI rule, then assumed it was satisfied because SSDI income is constant during the year. The facts never establish 24 months of entitlement, the duration input defaults to 0, and the 54-year-old head fails the age-65 test."
-us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"Described PolicyEngine's disability rule as triggered by any SSDI receipt. That rule requires 24 months of SSDI receipt, recorded as a separate input that defaults to 0 here, so the head qualifies only through the age-65 test and fails it at 54."
-us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Took the missing SSDI start date to mean receipt was ongoing and long enough to clear the 24-month waiting period. The prompt instead says unlisted numeric and status inputs are 0 or false, so months of SSDI receipt are 0 and the 54-year-old head is ineligible."
-us,scenario_074,head_medicare_eligible,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Explicitly assumed the 24-month SSDI waiting period was met, which contradicts the instruction not to infer unlisted facts. With SSDI receipt duration at the default 0, only the age-65 test applies, and the head, at 54, fails it."
-us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated SSDI income as proof that the disability waiting period was complete. The 24 months of entitlement are never established, the duration input defaults to 0, and the 54-year-old head does not meet the age-65 threshold."
-us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied the disability pathway based on SSDI receipt alone and skipped its 24-month entitlement requirement. That duration input is unlisted and therefore 0, so the pathway fails and the head, under 65, is ineligible."
-us,scenario_074,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Claimed PolicyEngine makes any SSDI recipient Medicare eligible. The engine's disability pathway requires 24 months of SSDI receipt, which defaults to 0 because it was not listed, leaving only the age-65 test, which the 54-year-old head fails."
-us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Equated being an SSDI recipient with being Medicare eligible and ignored the 24-month entitlement requirement. That requirement is unmet because receipt duration is unlisted and therefore 0, and at 54 the head fails the age-65 test."
-us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Granted Medicare because the head receives Social Security disability income, with no check of the 24-month waiting period. Months of SSDI receipt default to 0, so the disability pathway fails and the 54-year-old head is ineligible by age."
-us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Treated SSDI receipt as qualifying for Medicare on its own. Under-65 eligibility requires 24 months of SSDI entitlement, which the facts do not supply (duration defaults to 0), so the 54-year-old head is not eligible."
-us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Concluded Medicare eligibility from SSDI income alone and omitted the 24-month entitlement condition. That condition fails because the unlisted months-of-receipt input is 0, and the head, at 54, is below the age-65 threshold."
-us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated SSDI receipt as making the head Medicare eligible without applying the 24-month entitlement requirement. The receipt duration was not listed and defaults to 0, so only the age-65 test counts, and a 54-year-old fails it."
-us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"Asserted that PolicyEngine treats SSDI recipients as Medicare eligible regardless of age. The engine gates the disability pathway on 24 months of SSDI receipt, which defaults to 0 here, so the head is eligible only by age and fails at 54."
-us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Correctly noted that age 54 fails the age test, then treated disability income as meeting the disability pathway. That pathway requires 24 months of SSDI entitlement, and the unlisted duration input is 0, so the head is ineligible."
-us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"Read SSDI receipt for this year as satisfying the disability pathway. One constant tax year gives at most 12 months, short of the 24 months of entitlement required, and the unlisted duration defaults to 0, so the 54-year-old head is not eligible."
-us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"Claimed PolicyEngine grants Medicare for any positive Social Security disability income. The engine requires 24 months of SSDI receipt, a separate input that defaults to 0 here, so the head fails both the disability pathway and the age-65 test."
-us,scenario_074,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Keyed the PolicyEngine disability pathway on positive SSDI income, when the engine keys it on at least 24 months of SSDI receipt. That input was not listed and defaults to 0, so the 54-year-old head is ineligible."
-us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that SSDI receipt for the full year meets the 24-month waiting period. A single tax year supplies at most 12 months, and the prompt adds no prior receipt (duration defaults to 0), so the rule is unmet and the head, under 65, is ineligible."
-us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"Named the statutory waiting period but treated SSDI receipt as proof it had elapsed. The 24 months of entitlement are never given, the unlisted duration defaults to 0, and the 54-year-old head fails the age-65 test."
-us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"Treated full-year SSDI receipt as making the head a Medicare-eligible disabled beneficiary. Eligibility requires 24 months of SSDI entitlement, more than the one year the facts cover, and the duration input defaults to 0, so the head is ineligible at 54."
-us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"Explicitly assumed the 24-month waiting period was met because disability benefits are ongoing, which contradicts the instruction to treat unlisted status facts as false. With months of SSDI receipt at 0, only the age-65 test applies, and the head fails it."
-us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Social Security disability income as automatically qualifying for disability-based Medicare and skipped the 24-month entitlement requirement. That requirement fails because the unlisted receipt duration is 0, and the head, at 54, is below age 65."
-us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"Cited the SSDI waiting-period rule and then used the full-year constancy instruction to treat it as satisfied. Constancy covers only the 12 months of the tax year, not 24 months of entitlement, and the unlisted duration defaults to 0, so the 54-year-old head is not Medicare eligible."
+us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly saw that the head is under 65. It then assumed the 24-month SSDI waiting period was met because disability is 'constant for the full year.' But being constant this year says nothing about the 24 months before, and the unlisted months of SSDI receipt count as 0, so the disability route fails."
+us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model claimed that PolicyEngine grants Medicare whenever SSDI income is positive. PolicyEngine actually requires at least 24 months of SSDI receipt. That month count is unlisted and therefore 0, so a 54-year-old head with SSDI dollars is still ineligible."
+us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model reasoned that 'already receiving SSDI' means the 24-month waiting period has been met. The household gives no months of SSDI entitlement and that input defaults to 0, so the waiting period is not met and the age-65 test is the only route left, which the 54-year-old head fails."
+us,scenario_074,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model named the 24-month benefit requirement and then treated it as met without any supporting fact. Months of SSDI receipt are unlisted and therefore 0, so the head fails the disability route and, at 54, fails the age-65 route too."
+us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model assumed the 24-month SSDI waiting period was met because status is held constant for the year. Year-long status does not supply the 24 months of prior entitlement, the unlisted month count is 0, and the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model wrongly read 'constant disability income status throughout the year' as meeting the 24-month SSDI requirement. The unlisted months of SSDI receipt are 0, so Medicare depends on age alone, and a 54-year-old head is not eligible."
+us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's disability rule is triggered by receiving SSDI benefits at all. The rule actually requires at least 24 months of SSDI receipt. That count is unlisted and therefore 0, so the under-65 head is not Medicare eligible."
+us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model saw that no SSDI start date was given and then assumed long-ongoing receipt. The prompt says to treat unlisted numbers as 0, so months of SSDI receipt are 0, the 24-month requirement fails, and the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model openly assumed the 24-month SSDI waiting period was met, which contradicts the instruction to treat unlisted inputs as 0. With zero months of SSDI receipt and an age of 54, the head meets neither Medicare route."
+us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model mentioned the disability waiting period but treated it as satisfied just because SSDI income is present. The months of SSDI receipt are unlisted and therefore 0, so the waiting period is not met and the 54-year-old head is not Medicare eligible."
+us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the disability route on the strength of the SSDI dollar amount alone and skipped the 24-month entitlement requirement. That requirement is unmet because months of SSDI receipt default to 0, so the under-65 head is ineligible."
+us,scenario_074,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model treated receiving any SSDI income as automatic Medicare eligibility and never applied the 24-month SSDI receipt requirement. Months of receipt are unlisted and therefore 0, so the 54-year-old head is not eligible."
+us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model equated being an SSDI recipient with being Medicare eligible and left out the 24-month entitlement requirement. Months of SSDI receipt are unlisted and therefore 0, so only the age-65 test applies, and the 54-year-old head fails it."
+us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility based only on SSDI income and ignored the 24-month entitlement requirement. Months of SSDI receipt are 0 by default, so the 54-year-old head is not eligible."
+us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model said SSDI income alone qualifies the head for Medicare and skipped the 24-month waiting period. The unlisted month count is 0, so the waiting period is unmet, and at 54 the head fails the age-65 test."
+us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated receiving SSDI as sufficient for Medicare without checking that 24 months of entitlement had passed. That month count is 0, so the head qualifies only through age, and fails because they are 54."
+us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility directly from SSDI receipt and missed the 24-month entitlement requirement. Months of SSDI receipt are unlisted and therefore 0, so the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine treats anyone receiving SSDI as Medicare eligible regardless of age. PolicyEngine actually requires at least 24 months of SSDI receipt, which is unlisted and therefore 0, so the 54-year-old head is not eligible."
+us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly said age 54 does not qualify, but then concluded that the SSDI income does qualify, without applying the 24-month entitlement test. Months of SSDI receipt default to 0, so the disability route fails and the head is ineligible."
+us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt during the current year as satisfying the disability route. That route requires 24 months of prior SSDI entitlement, and the unlisted month count is 0, so the under-65 head is not Medicare eligible."
+us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine grants Medicare when SSDI income is positive. PolicyEngine actually tests months of SSDI receipt against a 24-month minimum. That input is unlisted and therefore 0, so the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied a made-up PolicyEngine rule triggered by positive SSDI dollars. The real test requires at least 24 months of SSDI receipt. That count defaults to 0 here, so the head fails the disability route and, at 54, the age-65 route too."
+us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed that 12 months of SSDI receipt in the year meets the 24-month waiting period. One year of receipt cannot meet a 24-month requirement, and the unlisted month count is 0 anyway, so the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model said receiving SSDI establishes Medicare eligibility 'after the statutory waiting period' and then assumed that period had already passed. Months of SSDI receipt are unlisted and therefore 0, so the waiting period is unmet and the under-65 head is not eligible."
+us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model concluded that a full year of SSDI makes the head a Medicare-eligible disabled beneficiary, but it ignored the 24-month entitlement requirement. That requirement is unmet because months of SSDI receipt default to 0, so the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model said the 24-month waiting period was 'assumed met since receiving ongoing disability benefits,' which adds a fact the prompt does not give. Months of SSDI receipt are 0 by default, so the under-65 head is not Medicare eligible."
+us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI income as qualifying for disability-based Medicare without applying the 24-month entitlement requirement. The unlisted month count is 0, so the requirement fails, and the 54-year-old head is ineligible."
+us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model mentioned the required waiting period and then counted it as met because facts are held constant for the year. Holding status constant this year does not supply 24 prior months of SSDI entitlement; that count is 0, so the 54-year-old head is not Medicare eligible."
us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model applied employee payroll-tax rates to non-wage income despite acknowledging that the household's income came from Social Security disability. It failed to set unlisted wages to zero and therefore invented a covered-earnings base that the household facts do not contain.
us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required output contract."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning uses the correct $16,100 standard deduction and 2026 brackets. It gets $13,843 of ordinary tax plus $934 of capital gains tax, for $14,777 total. It then submitted 12,523, a number that nothing in its own derivation supports."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"It gave a 62-year-old a $24,000 standard deduction by adding a $9,000 age/blindness amount. That add-on requires age 65+ or blindness, and disability alone does not qualify. It also started from the outdated $15,000 base instead of $16,100. It then subtracted an invented $5,252 'disability standard deduction adjustment', which no provision of the tax code supports."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the correct $16,100 deduction and $934 of preferential-rate tax, but estimated 2026 single bracket thresholds too low (12% band ending at $49,400 instead of $50,400, 10% band ending at $12,150 instead of $12,400). That pushed $1,000 more income into the 22% band and gave $13,948 of ordinary tax instead of $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied the pre-OBBBA $15,000 standard deduction instead of the 2026 amount of $16,100, which overstated taxable income by $1,100 ($94,286). It also taxed ordinary income on guessed inflation-adjusted brackets (12,250/49,750) instead of the actual 2026 thresholds (12,400/50,400)."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its reasoning computed exactly $14,777.10: $16,100 deduction, $13,843.20 of ordinary tax on $86,960, and $933.90 at 15%. It then discarded that figure as 'too high' and submitted an unsupported $6,710."
-us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $19,000 single standard deduction instead of $16,100, which understated taxable income at $90,286. It also used bracket thresholds of 12,200/49,550 instead of 12,400/50,400, and its explanation derives $14,228.10. The submitted $14,988 matches neither that derivation nor the correct $14,777."
-us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset for 2026: an $8,600 standard deduction, a $5,400 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA structure permanent, so the law is a $16,100 deduction, no exemption, and 10/12/22% brackets. Taxing $38,660 at 25% instead of 22% inflated ordinary tax to $16,605."
-us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied post-sunset pre-TCJA law: an $8,300 deduction, a $5,300 personal exemption, and 10/15/25% brackets. The 2026 law is the permanent TCJA structure with a $16,100 deduction, no exemption, and 12% and 22% brackets. This gave $16,778.75 of ordinary tax instead of $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed a TCJA sunset: a $13,600 combined standard deduction and personal exemption, with 10/15/25% brackets. 2026 law has a $16,100 standard deduction, no personal exemption, and 10/12/22% brackets. The sunset assumption inflated ordinary tax to about $17,315 instead of $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no numeric derivation. The correct path subtracts the $16,100 deduction to reach $93,186, taxes $86,960 of ordinary income on 2026 brackets (10% to $12,400, 12% to $50,400, then 22%) for $13,843, and adds $934 on $6,226 at 15%. Its $14,930.50 is $153 too high, which matches taxing ordinary income on thresholds or a deduction lower than the 2026 values."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA standard deduction expired and personal exemptions returned for 2026. OBBBA made the TCJA $16,100 standard deduction and 10/12/22% brackets permanent with no personal exemption. That produced the inflated $17,764."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: an $8,300 standard deduction, a $5,150 personal exemption, and 10/15/25% brackets, taxing $89,610 of ordinary income for $17,122. Under the 2026 law ($16,100 deduction, no exemption, 10/12/22% brackets), ordinary income is $86,960 and its tax is $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation taxes $93,186 of taxable income: $13,843 on $86,960 of ordinary income on 2026 brackets, plus $934 on $6,226 at 15%. Its $11,843 is $2,934 too low, which equals roughly $13,300 of 22%-bracket income escaping tax. That points to an overstated deduction or omitted income."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed a post-TCJA sunset: an $8,300 standard deduction, a $5,050 personal exemption, and 10/15/25% brackets, taxing $89,710 of ordinary income for $17,405. 2026 law keeps the TCJA structure: a $16,100 deduction, no exemption, and 10/12/22% brackets, giving $13,843 on $86,960."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 $15,000 standard deduction and the 2025 bracket thresholds (11,925/48,475) instead of 2026's $16,100 deduction and 12,400/50,400 thresholds. That overstated taxable income by $1,100 and pushed extra income into the 22% band, giving $15,221.10."
-us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It guessed a $15,400 standard deduction instead of the 2026 amount of $16,100, and bracket thresholds of 12,250/49,800 instead of 12,400/50,400. Ordinary taxable income became $87,660 with $14,060 of tax, instead of $86,960 with $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation taxes $93,186 of taxable income ($109,286 AGI less the $16,100 deduction): $13,843 on ordinary income plus $934 at 15%. Its $9,468 is $5,309 short, which at the 22% marginal rate means about $24,000 of taxable income dropped out of its base."
-us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed itemized medical deductions. The $600 of medical and OTC expenses is far below the 7.5%-of-AGI floor ($8,196), so the deductible medical amount is $0 and the $16,100 standard deduction applies. Its $1,446 also implies it taxed only about $14,000 of income, leaving most of the $90,020 taxable pension untaxed."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation is $13,843 of ordinary tax on $86,960 (after the $16,100 deduction, on 2026 10/12/22% brackets) plus $934 at 15% on $6,226. Its $13,361 is $1,416 short, which equals about $6,400 of 22%-bracket income escaping tax, for example through an extra deduction the 62-year-old does not qualify for."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset: about $8,344 of standard deduction plus a $5,322 personal exemption, taxing $89,394 on 10/15/25% brackets for $16,749. For 2026 the TCJA structure is permanent: a $16,100 deduction, no exemption, and 10/12/22% brackets."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law: an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets, giving $16,895 on $89,460 of ordinary income. 2026 law is a $16,100 deduction, no exemption, and 10/12/22% brackets, giving $13,843 on $86,960."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated a $15,400 standard deduction instead of the 2026 amount of $16,100, and bracket thresholds of 12,250/49,775 instead of 12,400/50,400. That taxed $87,660 of ordinary income for $14,062.70 instead of $86,960 for $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA-sunset law with an $8,522 standard deduction and 10/15/25% brackets. 2026 law uses a $16,100 deduction and 10/12/22% brackets. It also miscomputed taxable income as $100,764; $109,286 less $8,522 is $100,764 before removing nothing, yet it then taxed $94,538 of ordinary income at 25% above $50,928."
-us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $14,777.15 derived from $93,186 of taxable income."
-us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It invented about $18,608 of nonrefundable credits for 'age 62 senior credit and disability considerations'. The Credit for the Elderly or Disabled is capped at 15% of a $5,000 base, and that base is reduced by the $2,240 nontaxable pension and by half of AGI above $7,500, so it is fully eliminated at $109,286 AGI. It also used the 2025 $15,750 deduction instead of $16,100, and it mis-subtracted to get $103,536 of taxable income instead of $93,536."
-us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and the 2025 bracket thresholds (11,925/48,475) instead of 2026's $16,100 deduction and 12,400/50,400 thresholds. That taxed $87,310 of ordinary income for $14,122.20 instead of $86,960 for $13,843."
-us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It computed $20,146 of tax and stated that the Credit for the Elderly or Disabled yields no usable credit, yet it submitted $0. Tax before refundable credits equals the pre-credit tax when no nonrefundable credit applies. Along the way it also put the $2,240 tax-exempt pension into AGI, misadded AGI as $131,866, and gave a 62-year-old the over-65 standard deduction."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning gets it right: $16,100 standard deduction, $13,843 ordinary tax plus $934 at 15%, totaling $14,777. It then submitted $12,523, a number with no derivation that contradicts its own calculation."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied a $24,000 standard deduction that includes an age-65 add-on, even though the head is 62. It also used outdated brackets. It then subtracted an invented $5,252 'disability standard deduction adjustment'. Disability does not increase the standard deduction, and the Credit for the Elderly or Disabled is fully phased out at $109,286 AGI."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It submitted $0 while its explanation states $6,900, and neither figure was ever calculated. With no nonrefundable credits available, the $109,286 AGI less the $16,100 standard deduction produces $14,777 of tax, so a zero result treats a heavily taxed pension and wage income as untaxed."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction but estimated 2026 brackets of 10% to $12,150 and 12% to $49,400, instead of the actual $12,400 and $50,400. That overstated ordinary tax at $13,948 instead of $13,843, putting its answer $105 too high."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction (the pre-OBBBA 2025 amount) instead of the 2026 amount of $16,100. It also used guessed thresholds of 10% to $12,250 and 12% to $49,750 instead of $12,400 and $50,400. The extra $1,100 of ordinary income taxed at 22% plus the lower brackets produced $15,087."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed the exact $14,777.10 using the $16,100 deduction and the correct 2026 brackets. It then threw that result away as 'high' and replaced it with an unsupported $6,710 that no bracket calculation produces."
+us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It claimed a $19,000 2026 single standard deduction under the OBBBA; the actual amount is $16,100. It also used bracket thresholds of $12,200 and $49,550, which gave $14,228.10. It then submitted $14,988, which does not match its own calculation."
+us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, applying an $8,600 standard deduction plus a $5,400 personal exemption and 10/15/25% brackets. The OBBBA made the TCJA rates permanent: a $16,100 standard deduction, no personal exemption, and 10/12/22% brackets."
+us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the pre-TCJA structure for 2026: an $8,300 standard deduction plus a $5,300 personal exemption, with 10/15/25% brackets. The OBBBA made the TCJA permanent, so the correct inputs are the $16,100 standard deduction and the 10/12/22% brackets, which give $13,843 of ordinary tax instead of $16,779."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, deducting a combined $13,600 standard deduction and personal exemption and taxing ordinary income at 10/15/25%. Under the OBBBA's permanent TCJA rates the deduction is $16,100 with no exemption, and the brackets are 10/12/22%."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $14,930.50 is $153 above the correct figure. That gap equals about $700 of extra ordinary income taxed at 22%, which means it used a standard deduction of roughly $15,400 instead of the 2026 single amount of $16,100."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA standard deduction would expire and personal exemptions would return for 2026, taxing income at the old 10/15/25% brackets. The OBBBA made the $16,100 standard deduction and the 10/12/22% brackets permanent."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the TCJA sunset for 2026: an $8,300 standard deduction plus a $5,150 personal exemption, with 10/15/25% brackets. Under the OBBBA the standard deduction is $16,100, there is no exemption, and the brackets are 10/12/22%, which gives $13,843 of ordinary tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $11,843 is $2,934 below the correct total of $13,843 ordinary tax plus $934 at 15%. That shortfall is more than three times the entire preferential-rate tax, so it understated the ordinary tax, using brackets or a deduction more generous than the 2026 single parameters ($16,100 deduction; 22% bracket starting at $50,400)."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA sunset, using an $8,300 standard deduction plus a $5,050 personal exemption and 10/15/25% brackets. The OBBBA made the $16,100 standard deduction and the 10/12/22% brackets permanent."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and the 2025 brackets (10% to $11,925, 12% to $48,475) instead of the 2026 values: a $16,100 deduction and thresholds of $12,400 and $50,400. That overstated ordinary tax at $14,287 instead of $13,843."
+us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,400 instead of $16,100 and used bracket thresholds of $12,250 and $49,800 instead of $12,400 and $50,400. That pushed ordinary tax to $14,060 instead of $13,843."
+us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $9,468 is about $5,300 below the correct total of $13,843 ordinary tax on $86,960 plus $934 on $6,226 of preferential income. That is equivalent to leaving roughly $24,000 of pension and wage income untaxed at the 22% rate. No deduction beyond the $16,100 standard deduction and no nonrefundable credit applies."
+us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed itemized medical deductions, but the $600 of medical and over-the-counter costs is far below the 7.5%-of-AGI floor ($8,196), so the $16,100 standard deduction applies. Its $1,446 also treats the $90,020 taxable pension as if it were barely taxed, when it pushes ordinary income well into the 22% bracket."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $13,361 is $1,416 low, which equals removing about $6,400 of ordinary income at the 22% marginal rate. A 62-year-old gets only the $16,100 standard deduction: no age-65 add-on and no senior deduction. The full $86,960 of ordinary taxable income is therefore taxed at $13,843, plus $934 on the gains and dividends."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, applying a $5,322 personal exemption plus an $8,344 standard deduction and the 10/15/25% brackets. The OBBBA made the $16,100 standard deduction and the 10/12/22% brackets permanent, with no personal exemption."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the post-sunset structure: an $8,300 standard deduction plus a $5,300 personal exemption, with 10/15/25% brackets. Under the OBBBA's permanent TCJA rates, 2026 uses a $16,100 deduction and 10/12/22% brackets, which give $13,843 of ordinary tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,400 instead of $16,100 and used bracket thresholds of $12,250 and $49,775 instead of $12,400 and $50,400. Ordinary taxable income became $87,660 instead of $86,960, and ordinary tax $14,063 instead of $13,843."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, applying an $8,522 standard deduction and 10/15/25% brackets, and also dropped the personal exemption that the sunset regime it assumed would include. The OBBBA made the $16,100 deduction and the 10/12/22% brackets permanent."
+us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for this output. The correct calculation is $16,100 standard deduction, $13,843 of ordinary tax on $86,960, and $934 at 15% on $6,226 of gains and dividends, for $14,777."
+us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It made an arithmetic error: $109,286 minus $15,750 is $93,536, not the $103,536 it used. It then invented about $18,608 of nonrefundable 'senior/disability' credits. The Credit for the Elderly or Disabled is fully phased out at $109,286 AGI, and a 62-year-old has no age-based credit, so nothing reduces the $14,777 tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 and the 2025 brackets (10% to $11,925, 12% to $48,475) without the 2026 inflation adjustment to $16,100, $12,400 and $50,400. That overstated ordinary tax at $14,122 instead of $13,843."
+us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It wrongly added the $2,240 tax-exempt pension to AGI, misadded the total to $131,866, and applied an over-65 standard deduction to a 62-year-old. It then reported $0 even though it computed $20,146 of tax and itself found no usable Credit for the Elderly or Disabled, so no credit justified zeroing the liability."
us,scenario_075,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the listed disability as sufficient to activate a Florida disabled-adult Medicaid pathway without applying that pathway's complete eligibility requirements. The head receives no SSI and the engine assigns medicaid_category NONE, so no categorical pathway exists and Medicaid eligibility is 0."
us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"It treated the ""is disabled"" flag as automatic Medicare eligibility. The disability pathway requires 24 months of SSDI entitlement, and this head gets no Social Security disability benefits and has zero months of SSDI receipt. At 62, the head also fails the age-65 test, so the answer is not eligible."
@@ -5937,51 +6094,53 @@ us,scenario_075,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False
us,scenario_075,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"It applied a disability-based Medicare rule keyed to the ""is disabled"" flag. The actual pathway requires 24 months of SSDI entitlement, and the head has zero SSDI months. With no ESRD and age below 65, eligibility is 0."
us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"It stated the rule as ""age 65+ or disabled"", dropping the requirement that under-65 disability eligibility rest on 24 months of SSDI receipt. The head receives no Social Security disability benefits, so only the age test applies, and it fails at 62."
us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"It concluded that being 62 and disabled makes the head Medicare-eligible. Medicare needs age 65+, or 24 months of SSDI entitlement or ESRD for younger people. The head has no SSDI and no ESRD, so they are not eligible."
-us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the correct 6.2% Social Security and 1.45% Medicare rates but miscalculated their combined application to wages. The PolicyEngine component amounts are $784.69 and $183.52, which sum to $968.21 rather than $966."
-us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the stated FICA rates incorrectly: 6.2% plus 1.45% on $12,656 does not produce $1,054. The component calculation yields $784.69 of Social Security tax and $183.52 of Medicare tax, totaling $968.21."
-us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
-us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model's own component calculation produced $968.18 before engine-level precision, but it then changed that result to $989 under the label of rounding. Rounding cannot turn approximately $968.2 into $989; the PolicyEngine components total $968.21."
+us,scenario_075,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"The model's explanation applies 6.2% Social Security plus 1.45% Medicare to the $12,656 in wages and ends with 'value = 968', which is the correct derivation. The recorded answer is $0.00, so the numeric value field did not match the explanation's 968. This is a submission/parse failure, not a tax error."
+us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly limited the base to $12,656 in wages, used 6.2% plus 1.45%, and excluded Additional Medicare Tax. It then miscalculated the total: $784.67 + $183.51 = $968.18, not the $966 it reported."
+us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model used the right base ($12,656 in wages) and the right rates (6.2% Social Security plus 1.45% Medicare), which yield $968.18. Its $1,054 implies an effective rate of about 8.33%, so it miscalculated the rate-times-base step."
+us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax. The correct derivation is 7.65% of $12,656 in wages, which is about $968."
+us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed $784.67 of Social Security tax plus $183.51 of Medicare tax, for a total of $968.18. It then reported $989, citing 'rounding', which contradicts its own correct total."
us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely extended Idaho WIC eligibility through age 8 even though WIC child eligibility ends at the fifth birthday. It also divided or expressed the income ratio incorrectly: using its own $23,030 poverty guideline, $173,820 is about 755% of poverty and exceeds the 185% limit by a wide margin; medical expenses do not establish WIC eligibility or cure either failure."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It set up AGI $164,220, the $800 QBI deduction and taxable income of about $139,120 correctly, and its own bracket computation gave about $24,321. It then replaced that with an unexplained ""tentative tax of $29,226"" (roughly $4,900 higher) before subtracting the $4,600 of credits. The correct pre-credit tax is $24,211.80 at the 2026 HoH brackets, so the result should have been $19,611.80."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its taxable income ($139,270) and credits ($4,400 CTC + $200 CDCC) are exactly right, but its bracket tax of $24,532.80 uses the 2025 HoH thresholds (10% to $17,000, 12% to $64,850, 22% to $103,350). The inflation-indexed 2026 thresholds are $17,700 / $67,450 / $105,700, which give $24,211.80, so it overstated tax by $321."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $9,600 of child support to gross income, although child support is not taxable. It also subtracted the $6,480 ESI premium as an above-the-line deduction and used an invented $26,050 HoH standard deduction with fabricated brackets (10% to $20,550, 12% to $78,100). On credits, it applied $2,000 per child instead of $2,200, dropped the $200 CDCC, and never took the $800 QBI deduction; offsetting errors produced $18,863."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It itemized $37,900 by imputing $29,400 of mortgage interest from the $420,000 balance and $8,500 of state tax. No mortgage interest is listed, so it counts as $0, and the $24,150 HoH standard deduction applies. It also used 2025-era brackets and a $2,000-per-child CTC. It then discarded its own computed $17,033 for an unsupported $21,889."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized about $33,200 by imputing $25,200 of mortgage interest from the $420,000 loan balance plus $8,000 of SALT. Unlisted mortgage interest is $0, so the $24,150 HoH standard deduction governs. It also used a $2,000-per-child CTC and miscomputed the 22% band. It then abandoned its own $14,306 result for an arbitrary $21,625."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated most of the $4,400 CTC as refundable and counted only about $1,000 as nonrefundable. Because pre-credit tax (about $24,000) far exceeds the credit, the entire $4,400 CTC is nonrefundable and none is refundable ACTC. It also used a $24,500 standard deduction instead of $24,150 and 2025 bracket thresholds instead of the 2026 HoH brackets."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $800 Section 199A QBI deduction on the $4,000 of rental income, leaving taxable income at $140,070 instead of $139,270. That adds $192 of tax at the 24% rate ($24,403.80 vs $24,211.80). Its standard deduction, 2026 brackets and $4,600 of credits were otherwise correct."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It filed the unmarried parent of two qualifying children as single instead of head of household. It also itemized $37,300 by imputing $27,300 of mortgage interest from the loan balance plus $10,000 SALT, although no mortgage interest is an input and the $24,150 HoH standard deduction applies. It then taxed the result at 2025 single brackets and used a $2,000-per-child CTC instead of $2,200."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It filed as single with the single standard deduction ($16,100) and single brackets instead of head of household ($24,150 and HoH brackets). It also cut wages by the $6,480 ESI premium, although AGI includes the full $160,000. It used a $2,000-per-child CTC and then rounded $22,315 to $22,300."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its bracket tax of about $24,725 on $140,070 matches 2025 HoH thresholds ($17,000 / $64,850 / $103,350) rather than the 2026 thresholds ($17,700 / $67,450 / $105,700). It also omitted the $800 QBI deduction on rental income, so it taxed $140,070 instead of $139,270."
-us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It declared the CTC ""fully refundable in 2026"" and subtracted $0 of it. The CTC first offsets income tax as a nonrefundable credit, so the full $4,400 reduces this $24,211.80 liability. It also used a $23,000 standard deduction, omitted the $800 QBI deduction and overstated bracket tax at $27,259."
-us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 ESI premium to get AGI of $157,740, but AGI includes the full $160,000 of wages, for $164,220. It also used a $23,200 standard deduction instead of $24,150, omitted the $800 rental QBI deduction, and applied a $2,000-per-child CTC instead of $2,200."
-us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law to 2026, using a $12,155 HoH standard deduction, three personal exemptions and a CTC it called fully phased out. The OBBBA made the TCJA structure permanent: a $24,150 HoH standard deduction, no personal exemptions and a $2,200 CTC unreduced below $200,000 AGI. It also subtracted the ESI premium from wages. Its submitted $26,346.25 does not even match its own stated $24,775.75."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, so it used an $11,687 standard deduction plus three $5,062 exemptions and a $1,000-per-child CTC. 2026 law instead provides a $24,150 HoH standard deduction, no exemptions and a $2,200-per-child CTC. It also cut wages by the ESI premium and omitted the $800 QBI deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It started from the correct $164,220 AGI but subtracted a $4,000 CTC ($2,000 per child) instead of the 2026 $2,200-per-child $4,400. It never subtracted the $200 CDCC. Its implied pre-credit tax of $22,985 also understates the $24,211.80 from the 2026 HoH brackets on $139,270 of taxable income."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expiration, applying a $12,090 standard deduction, $15,900 of personal exemptions and pre-TCJA brackets, and it gave no CTC at all. 2026 law keeps the $24,150 HoH standard deduction, the 10/12/22/24 brackets and a nonrefundable $2,200-per-child CTC worth $4,400 here. It also subtracted the ESI premium from wages."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied ""2026 sunset"" law, with personal exemptions and a smaller standard deduction, and declared the CTC ""fully phased out due to high income"" under the old $75,000 threshold. Under 2026 law the CTC phase-out starts at $200,000, so the full $4,400 applies against tax computed with the $24,150 HoH standard deduction. It also cut AGI to $157,740 by subtracting the ESI premium."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. The correct path is taxable income of $139,270 ($164,220 - $24,150 - $800), tax of $24,211.80 at 2026 HoH brackets, less $4,600 of CTC and CDCC. Its $19,535 implies pre-credit tax of $24,135 after the same $4,600 of credits, a $76.80 bracket-arithmetic understatement."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It computed tax on ""gross income after pre-tax health insurance deductions,"" subtracting the $6,480 ESI premium from wages. AGI includes the full $160,000 of wages ($164,220 total). Its $21,743 is $2,131 above the correct $19,611.80, consistent with a pre-credit tax overstatement or undercounted nonrefundable credits on top of the wage error."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It treated the CTC as phased out, a pre-TCJA-sunset assumption, and subtracted only the $200 CDCC. Under 2026 law the $2,200-per-child CTC phases out only above $200,000 AGI, so the full $4,400 is nonrefundable here. It also computed tax on an AGI of $157,740 that wrongly subtracts the $6,480 ESI premium."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $22,500 HoH standard deduction instead of the 2026 $24,150 and omitted the $800 QBI deduction on rental income, overstating taxable income at $141,720 instead of $139,270. It also applied a $2,000-per-child CTC ($4,000) instead of $2,200 ($4,400)."
-us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,480 ESI premium from wages to get AGI of $157,740 instead of $164,220. It also used a $23,100 standard deduction instead of $24,150, approximate brackets instead of the 2026 HoH thresholds, and a $2,000-per-child CTC instead of $2,200."
-us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2025 HoH standard deduction ($23,625) instead of the 2026 $24,150. It overstated bracket tax at $24,965 on $139,795 (2026 HoH brackets give $24,337.80 on that amount). It also applied an invented $2,250-per-child CTC instead of the $2,200 statutory amount."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its reasoning invokes a ""standard/itemized deduction"" choice. No mortgage interest or other itemizable expense is an input, so the $24,150 HoH standard deduction applies, giving tax of $24,211.80 less $4,600 of credits. Its $14,143 is $5,469 too low, which requires roughly $22,800 of extra deductions at the 24% rate, consistent with itemizing deductions absent from the inputs."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It explicitly treated the $9,600 of child support received as taxable income, although child support is excluded from gross income. It also referenced mortgage-interest and state-tax adjustments that have no inputs. It never subtracted the $4,400 CTC and $200 CDCC against pre-credit tax of $24,211.80, so it landed at $28,930."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 employee ESI premium, giving AGI of $157,740 instead of $164,220, which understated taxable income and tax. It also applied a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 ($4,400)."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It computed tax as married filing jointly with the $32,200 MFJ standard deduction and MFJ brackets. The head is unmarried with two qualifying children and files head of household ($24,150 standard deduction, HoH brackets). It also omitted the $800 QBI deduction: $132,020 at MFJ brackets gives $18,468.40, less $4,600, which is exactly its $13,868.40."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It followed the correct structure (HoH, standard deduction, nonrefundable CTC and CDCC) but gave no figures. Its $19,606 implies pre-credit tax of $24,206 instead of $24,211.80 on $139,270 of taxable income, a $5.80 bracket-arithmetic shortfall."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It computed tax after ""deductions and exemptions,"" applying pre-TCJA personal exemptions that do not exist in 2026. The correct base is the $24,150 HoH standard deduction plus $800 QBI with no exemptions, then 2026 brackets and a full $4,400 CTC. Its $23,250 is $3,638 too high, consistent with losing most of the CTC under sunset-era rules."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used an understated $23,063 HoH standard deduction instead of $24,150 and a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 ($4,400). Its $24,570 bracket tax also exceeds what the 2026 HoH thresholds produce on its own $140,357."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: a $12,450 standard deduction, three $5,300 exemptions, pre-TCJA brackets and a CTC ""fully phased out above 75000 AGI."" 2026 law keeps the $24,150 HoH standard deduction, 10/12/22/24 brackets, and a $2,200-per-child CTC phasing out only above $200,000. It also omitted the $800 QBI deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed ""post-TCJA-expiration law,"" using three $5,300 exemptions, a $12,250 standard deduction, 10/15/25 brackets and a $1,000 CTC phased out above $75,000. The TCJA structure is permanent in 2026: a $24,150 HoH standard deduction and a $2,200-per-child CTC fully available at $164,220 AGI. It also omitted the $800 QBI deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied TCJA-expiration rules (a $12,585 standard deduction, $16,353 of personal exemptions, inflated 2017 brackets) and allowed no CTC. 2026 law gives a $24,150 HoH standard deduction, no exemptions and a nonrefundable $4,400 CTC. It also cut AGI by subtracting the $6,480 ESI premium from wages."
-us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It subtracted only ""about $1,000 nonrefundable CTC portion"" for two children, treating the rest as refundable. With about $24,200 of pre-credit tax, the entire $4,400 CTC is used as a nonrefundable credit. That leaves $19,611.80 rather than its $23,300."
-us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_income_tax_before_refundable_credits, so no answer could be scored."
-us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its AGI, $24,150 standard deduction, $800 QBI deduction, $139,270 taxable income and $4,600 of credits are all correct. Its bracket tax of $24,332.80 misapplies the 2026 HoH thresholds (10% to $17,700, 12% to $67,450, 22% to $105,700, 24% above), which give $24,211.80, so it overstated tax by $121."
-us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It invented $1,815 of rental depreciation from the $49,921 unadjusted basis of qualified property, shrinking rental income to $2,185 and the QBI deduction to $437. That basis only feeds the 199A UBIA limit; the $4,000 rental income stands and the QBI deduction is $800. It also used a $23,200 standard deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200."
-us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It split the $4,400 CTC into $3,400 refundable and $1,000 nonrefundable. The refundable ACTC applies only to CTC exceeding tax liability, and here liability far exceeds the credit, so all $4,400 is nonrefundable. It also omitted the $800 QBI deduction, taxing $140,070 instead of $139,270."
-us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed as married filing jointly (with a wrong $16,600 standard deduction) and MFJ brackets for an unmarried parent who files head of household with a $24,150 standard deduction and HoH brackets. It also omitted the $800 QBI deduction and used a $2,000-per-child CTC instead of $2,200."
-us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It added the $9,600 of nontaxable child support to AGI and treated the children as $1,700 other-dependent credits. It also declared the CTC phased out above $400,000 even though its own AGI was $169,820. It then submitted $18,779.30, which matches neither its computed $14,152.40 nor the correct $24,211.80 - $4,400 CTC - $200 CDCC = $19,611.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The setup was nearly right, with taxable income of about $139,120 and bracket tax of about $24,321. The model then swapped in an unexplained 'tentative tax' of $29,226, inflating the result by about $5,000. Using the exact 2026 head-of-household thresholds on $139,270 gives $24,211.80, and subtracting $4,600 in credits leaves $19,611.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"AGI, the $24,150 deduction, the $800 QBI deduction, $139,270 of taxable income and the $4,600 in credits were all correct. The model then used 2025 head-of-household brackets (10% to $17,000, 12% to $64,850, 22% to $103,350) instead of the 2026 thresholds ($17,700 / $67,450 / $105,700). That overstated bracket tax by $321 ($24,532.80 instead of $24,211.80)."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model added the nontaxable $9,600 of child support to gross income and subtracted the $6,480 ESI premium from wages. It used a made-up $26,050 standard deduction and brackets with no 24% rate, applied a $2,000-per-child CTC instead of $2,200, and left out the QBI deduction and the $200 CDCC. Its $18,863 is the product of these offsetting errors."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model gave only a rough estimate of $30,500, which is higher than even the correct pre-credit tax of $24,211.80. That means it never properly applied the $24,150 standard deduction, the $800 QBI deduction, the 2026 head-of-household brackets, or the $4,600 of nonrefundable CTC and CDCC."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated mortgage interest (about $29,400) from the $420,000 balance and added an estimated SALT amount so it could itemize. The prompt says not to infer unlisted expenses, so mortgage interest is $0 and the $24,150 standard deduction applies. It also used 2025 brackets and a $2,000-per-child CTC, then replaced its own computed $17,033 with an unexplained $21,889."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated about $25,200 of mortgage interest from the loan balance plus an estimated state tax deduction, so it itemized $33,200. Mortgage interest is $0 under the facts given, so the $24,150 standard deduction applies. It also used a $2,000-per-child CTC and then dropped its computed $14,306 for an unsupported $21,625."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model treated $1,700 per child of the CTC as refundable and so applied only about $1,000 of nonrefundable CTC. With roughly $24,000 of liability, the full $4,400 CTC is used nonrefundably. It also used a $24,500 standard deduction and 2025 brackets instead of the 2026 values ($24,150 and thresholds of $17,700 / $67,450 / $105,700)."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the §199A QBI deduction of $800 (20% of the $4,000 qualified rental income). That put taxable income at $140,070 instead of $139,270 and overstated tax by $192 (24% × $800). Everything else, including the 2026 brackets and the $4,600 in credits, was correct."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model filed the unmarried parent of two qualifying children as single instead of head of household and used 2025 single brackets. It itemized $37,300 by estimating $27,300 of mortgage interest from the balance plus $10,000 of SALT, although the prompt lists no interest or tax payments. It also used $2,000 per child for the CTC instead of $2,200."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used single status (a $16,100 deduction and single brackets) instead of head of household with the $24,150 deduction. It also subtracted the $6,480 ESI premium from the stated gross wages and used $2,000 per child for the CTC instead of $2,200."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $800 QBI deduction on the rental income, so it taxed $140,070 instead of $139,270. It also used 2025 head-of-household brackets, giving $24,724.80 instead of the $24,211.80 that the 2026 thresholds produce on $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"The model claimed the CTC is fully refundable in 2026 and so applied $0 of it nonrefundably. In fact the $4,400 CTC first offsets income tax and is fully used here. It also used a $23,000 deduction with no QBI deduction and overstated bracket tax at $27,259 instead of $24,211.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated $160,000 gross wages, giving an AGI of $157,740 instead of $164,220. It also used a $23,200 deduction instead of $24,150, omitted the $800 QBI deduction, and used a $2,000-per-child CTC instead of $2,200."
+us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and applied a pre-TCJA standard deduction, three personal exemptions and old brackets, and it treated the CTC as fully phased out. 2026 law keeps the $24,150 head-of-household deduction, no personal exemptions and a $2,200-per-child CTC. It also subtracted the ESI premium from wages, and its submitted $26,346.25 does not even match its own $24,775.75."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026, using a $11,687 deduction, personal exemptions and a $1,000-per-child CTC. 2026 law keeps the $24,150 head-of-household deduction, no exemptions, the 10/12/22/24% brackets and a $2,200-per-child CTC. It also subtracted the $6,480 ESI premium from gross wages."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a $2,000-per-child CTC ($4,000) instead of the 2026 amount of $2,200 ($4,400) and left out the $200 CDCC. Its $18,985 implies pre-credit tax of $22,985, about $1,227 below the $24,211.80 that the 2026 head-of-household brackets give on $139,270 after the $800 QBI deduction."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026, applying personal exemptions, a $12,090 deduction and pre-TCJA brackets, and it allowed no CTC. 2026 law keeps the $24,150 deduction, the 10/12/22/24% brackets and a $4,400 CTC that is fully nonrefundable here. It also subtracted the ESI premium from the stated gross wages."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied TCJA-expiration ('sunset') law, with personal exemptions, pre-TCJA brackets and a CTC fully phased out at a $75,000 threshold. 2026 law gives a $2,200-per-child CTC that phases out above $200,000, so the full $4,400 applies. It also subtracted the $6,480 ESI premium from gross wages."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $19,535 implies pre-credit tax of $24,135 after the same $4,600 in credits, which is $76.80 below the exact result. Applying the 2026 head-of-household thresholds ($17,700 / $67,450 / $105,700) to $139,270 gives $24,211.80, so the model approximated the bracket tax instead of computing it exactly."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced income for 'pre-tax health insurance deductions', but the $160,000 is stated as gross wages and the engine's AGI is $164,220. Its $21,743 is $2,131 too high, which means it did not apply the full $4,400 CTC plus $200 CDCC against the $24,211.80 bracket tax on $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated the CTC as phased out, which follows from assuming the TCJA expires, and applied only the $200 CDCC. In 2026 the CTC is $2,200 per child with a $200,000 phase-out threshold, so the full $4,400 is nonrefundable. It also subtracted the ESI premium from wages, giving an AGI of $157,740 instead of $164,220."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used outdated 2025 parameters: a $22,500 head-of-household deduction instead of $24,150, 2025 brackets, and a $2,000-per-child CTC instead of $2,200. It also left out the $800 QBI deduction on the rental income."
+us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated gross wages, giving an AGI of $157,740 instead of $164,220. It used a $23,100 deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200, and its errors offset to $19,279.60."
+us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $23,625 deduction (the 2025 figure) instead of $24,150 and a $2,250-per-child CTC instead of $2,200. Its bracket tax of $24,965 on $139,795 is overstated; the 2026 head-of-household thresholds give $24,211.80 on the correct $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no figures. Its $14,143 implies pre-credit tax of about $18,743, roughly $5,470 below the $24,211.80 that 2026 head-of-household brackets give on $139,270. That points to an overstated deduction (such as itemizing estimated mortgage interest) or joint-width brackets in place of the $24,150 standard deduction and head-of-household schedule."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model counted the $9,600 of child support as taxable income; child support is excluded from gross income. It also did not subtract the $4,400 CTC and $200 CDCC from the $24,211.80 bracket tax on $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated gross wages, giving an AGI of $157,740 instead of $164,220. It also applied a $2,000-per-child CTC ($4,000) instead of the 2026 amount of $2,200 ($4,400)."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status, with the $32,200 deduction and joint brackets, for an unmarried parent who files as head of household. Head-of-household status uses the $24,150 deduction and narrower brackets. It also left out the $800 QBI deduction."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The method was right (head of household, standard deduction, $4,600 in credits), but the answer is $5.80 low. That means the bracket tax was rounded to about $24,206 instead of computed exactly. The exact 2026 head-of-household bracket computation on $139,270 gives $24,211.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model applied personal exemptions, which follows from assuming the TCJA expires in 2026. 2026 law has no personal exemptions, a $24,150 head-of-household deduction, the 10/12/22/24% brackets and a $2,200-per-child nonrefundable CTC."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $23,063 deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200. Its bracket tax of about $24,570 overstates the $24,211.80 that 2026 head-of-household brackets give on $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law: a $12,450 deduction, $15,900 of personal exemptions, pre-TCJA brackets and a CTC phased out above $75,000. 2026 law keeps the $24,150 deduction, no exemptions, the 10/12/22/24% brackets and a $2,200-per-child CTC that phases out only above $200,000."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law: $5,300 exemptions, a $12,250 deduction, 10/15/25% brackets and a $1,000 CTC fully phased out. 2026 law keeps the $24,150 deduction, the 10/12/22/24% brackets and the full $4,400 CTC, which is nonrefundable here."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law, using personal exemptions, a $12,585 deduction, 2017-style brackets and no CTC. 2026 law keeps the $24,150 deduction and a $4,400 CTC. It also subtracted the $6,480 ESI premium from the stated gross wages."
+us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model applied only about $1,000 of nonrefundable CTC, treating the rest as refundable. With $24,211.80 of liability, the full $4,400 CTC is used nonrefundably, along with the $200 CDCC. Its rounded bracket tax of about $24,500 also overstates the exact figure."
+us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score."
+us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model got AGI, the $24,150 deduction, the $800 QBI deduction, $139,270 of taxable income and the $4,600 in credits right. It then reported bracket tax of $24,332.80, which is $121 above the $24,211.80 that the 2026 head-of-household thresholds ($17,700 / $67,450 / $105,700) give."
+us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a $1,815 depreciation deduction ($49,921 / 27.5) against rental income. The unadjusted basis of qualified property is only a §199A limit input, so rental income stays $4,000 and the QBI deduction is $800. It also used a $23,200 deduction instead of $24,150 and $2,000 per child for the CTC instead of $2,200."
+us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model split the CTC into $3,400 refundable and $1,000 nonrefundable. The full $4,400 CTC is first applied against the $24,000+ liability and is entirely nonrefundable here. It also left out the $800 QBI deduction, taxing $140,070 instead of $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status with a made-up $16,600 deduction for an unmarried parent who files as head of household (deduction $24,150). It also left out the $800 QBI deduction and used $2,000 per child for the CTC instead of $2,200."
+us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted nontaxable child support in AGI and used an invented $24,800 deduction and brackets. It declared the CTC fully phased out using a $400,000 threshold that its own $169,820 AGI does not exceed. It then submitted $18,779.30, which does not match its computed $14,152.40."
us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied the married-filing-jointly $400,000 phase-out threshold and a $2,000-per-child CTC instead of the head-of-household $200,000 threshold and 2026's $2,200 per child, then — after correctly stating that ACTC equals the CTC in excess of tax liability — ignored that offset even though pre-credit tax above $25,000 absorbs the entire $4,400 CTC nonrefundably. Its $2,400 is not the output of any rule it named; it is an unexplained reduction of the $3,400 ACTC ceiling, while the excess-over-liability computation yields $0."
us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It stated the governing rule — the refundable ACTC is 'claimed to the extent CTC exceeds tax used' — and then contradicted it by reporting the $1,700-per-child cap of $3,400 as the refundable amount. With taxable income near $140,000 and pre-credit federal tax over $25,000, the full $4,400 CTC is consumed as a nonrefundable credit, leaving zero excess to convert into refundable_ctc."
us,scenario_076,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It treated the entire Child Tax Credit as refundable, calling it a 'refundable child tax credit of 2,000 per child,' which errs twice: the 2026 per-child credit is $2,200, and only the portion unused against tax liability is refundable, capped at $1,700 per child. Because this filer's federal tax before credits exceeds $25,000, all $4,400 of CTC is applied nonrefundably and the refundable component is $0, not $4,000."
@@ -6002,120 +6161,123 @@ us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model
us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $4,000 of rental income as earnings from self-employment and invented a 30% expense deduction that the prompt did not provide. Rental income is not subject to self-employment tax here, leaving a zero tax base and $0 liability."
us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no self_employment_tax output or explanation, violating the required structured-output contract."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 and never subtracted the $9,622 head-of-household zero-rate amount. It also missed the $1,000 Idaho dependent care subtraction and took off a $410 child credit that the reference does not apply. It then moved the result to an unexplained $7,099."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It got the $1,000 dependent care subtraction right but left out the $800 QBI deduction, so its taxable income was $139,070 instead of $138,270. It then taxed that whole amount at 5.3% without subtracting the $9,622 head-of-household zero-rate amount, and took off a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted $9,600 of non-taxable child support as income and subtracted the $6,480 ESI premiums. It used the $15,000 single standard deduction instead of the $24,150 head-of-household amount, and then applied made-up graduated brackets instead of Idaho's flat 5.3% above $9,622. These errors produced an arbitrary $9,400."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It used the old 5.695% rate instead of the 2026 rate of 5.3%, and a $23,625 standard deduction. It dropped its own $4,673 threshold, so the $9,622 head-of-household zero-rate amount was never applied. It also missed the dependent care and QBI subtractions and ended on an unexplained $8,204."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used the old 5.695% rate, an understated $22,500 head-of-household standard deduction, and a $3,420 threshold instead of $9,622. After working out $7,876, it reported $9,112, a number its own arithmetic does not support."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It applied 5.3% to $139,720 using a made-up $24,500 standard deduction, with no zero-rate amount, and skipped the $1,000 dependent care and $800 QBI subtractions. It also said the credits lower the tax, yet reported $7,480, which is above its own $7,405 pre-credit figure."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its taxable income of $140,070 leaves out Idaho's $1,000 household and dependent care subtraction and the $800 QBI deduction. It used $9,346 as the zero-rate amount instead of $9,622. It then subtracted a $410 child credit that the reference computation does not include, which left it about $300 low."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It filed the head as single, using a $15,000 deduction, even though the head qualifies as head of household with a $24,150 deduction. It made up Idaho personal exemptions of $4,714 each, although Idaho has none, and used the repealed 5.695% rate. It also claimed a nonexistent $100 Idaho child care credit instead of the $1,000 dependent care subtraction, and never applied the $9,622 zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It used the old 5.695% rate and a $16,000 standard deduction instead of the $24,150 head-of-household amount, with no zero-rate amount. It then raised its own $8,443 figure to $9,400 with no computation behind the change."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It taxed all of $140,070 at 5.3% without subtracting the $9,622 head-of-household zero-rate amount. It also missed the $1,000 dependent care subtraction and the $800 QBI deduction, and took off a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It used the repealed 5.8% rate instead of 5.3%, and an understated $19,900 standard deduction instead of $24,150. It never applied the $9,622 zero-rate amount or the dependent care and QBI subtractions."
-us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It lowered AGI to $157,740 by subtracting the $6,480 ESI premiums, which the reference does not take out of AGI. It used a $23,200 deduction and the repealed 5.8% rate, and never applied the $9,622 zero-rate amount or the dependent care and QBI subtractions."
-us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted the $6,480 ESI premiums from AGI and used a $12,155 head-of-household deduction, about half the correct $24,150. It applied the old 5.695% rate with no zero-rate amount, and reported $8,247.54 even though its own arithmetic gave $8,291.07."
-us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a 5.0% rate instead of Idaho's 2026 rate of 5.3%, and did not subtract the $9,622 zero-rate amount. It also skipped the $1,000 dependent care and $800 QBI subtractions, and took off a $410 child credit plus a $40 dependent care credit, neither of which the reference applies."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used the repealed 5.8% rate on an unexplained $130,867 taxable income, with no $9,622 zero-rate amount. It then subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation. Its $8,100 answer matches applying an outdated rate of about 5.7–5.8% to roughly $140,000 without the $9,622 zero-rate amount, while the correct result is 5.3% × ($138,270 − $9,622) = $6,818.34."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to $129,750 of taxable income with no $9,622 zero-rate amount. The taxable income itself is understated and has no stated derivation, while the correct base is $138,270 taxed at 5.3%."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It used an outdated 5.8% rate instead of 5.3% and never subtracted the $9,622 head-of-household zero-rate amount. It also subtracted a child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. Its $7,322 is about what 5.3% on the full $138,270 comes to ($7,328), so it left out the $9,622 head-of-household zero-rate amount, which is worth $510 of tax."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $6,985 matches 5.3% on about $139,300 of taxable income, minus a $410 child credit. That means no $9,622 zero-rate amount, a missing $1,000 dependent care subtraction, and a child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation. Its $7,665 is $847 above the correct $6,818.34, which matches taxing a base of about $140,000 or more (5.3% gives about $7,400) with no $9,622 zero-rate amount and without the dependent care and QBI subtractions."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It wrongly said Idaho income tax is not modeled and answered $0. Idaho levies a 5.3% flat tax on taxable income above $9,622 for head-of-household filers, which here comes to $6,818.34."
-us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to all of $141,720, using an understated $22,500 standard deduction. It had no $9,622 zero-rate amount and no $1,000 dependent care or $800 QBI subtractions."
-us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It took the standard deduction twice: it started from federal taxable income of $139,795, which already reflects the deduction, and then subtracted about $23,625 more, giving $116,170. It also left out the zero-rate amount and subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave no derivation. Its $6,494 is $324 below the correct figure, which matches subtracting a $410 child credit that the reference does not include. The correct result is just 5.3% × ($138,270 − $9,622) = $6,818.34."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It based its estimate on 'itemized medical assumptions' instead of the $24,150 head-of-household standard deduction. Its $7,450 matches taxing roughly all of taxable income at 5.3% without the $9,622 zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It taxed all of $140,870 at 5.3%, using an understated $23,350 deduction, with no $9,622 zero-rate amount. It left out the dependent care and QBI subtractions and took off $346 in child credits that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed all of an understated $132,020 at 5.3% without the $9,622 zero-rate amount. It then subtracted a made-up $440 child credit, which the reference computation does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used a $4,930 zero-rate amount, which is the single-filer level, instead of the $9,622 head-of-household amount. It worked from $139,270 without the $1,000 dependent care subtraction and took off a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 and never subtracted the $9,622 head-of-household zero-rate amount. It also missed Idaho's $1,000 household and dependent care subtraction."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It wrongly subtracted the $6,480 employer insurance premiums, which dropped taxable income to $131,790 instead of $138,270. It used $9,960 as the zero-rate amount instead of $9,622 and subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 with no $9,622 head-of-household zero-rate amount and missed the $1,000 dependent care subtraction. It then took off a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It taxed all of $139,270 at 5.3% without subtracting the $9,622 zero-rate amount and without the $1,000 dependent care subtraction. It then subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It got taxable income right at $138,270 but used a $4,980 single-level zero-rate amount instead of the $9,622 head-of-household amount. It then subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It guessed a 4% effective rate instead of working the calculation. The correct figure comes from applying Idaho's flat 5.3% to $138,270 minus the $9,622 head-of-household zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It wrongly said Idaho does not follow the federal QBI deduction; Idaho allows the $800 QBI deduction here. It also missed the $1,000 dependent care subtraction, used the repealed 5.695% rate, and left out the $9,622 zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $12,450 head-of-household standard deduction instead of $24,150, which pushed taxable income up to $151,770. It applied the repealed 5.695% rate with no $9,622 zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to all of $136,070, instead of 5.3% on the amount above the $9,622 head-of-household zero-rate threshold."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It lowered AGI to $157,740 by subtracting the ESI premiums and used a $12,585 deduction instead of $24,150. It then applied the repealed 5.8% rate to the whole amount with no zero-rate amount."
-us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to a taxable income of $135,000–$140,000 with no $9,622 head-of-household zero-rate amount, which overstates the tax by roughly $400 or more compared with the $6,818.34 correct figure."
-us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to grade."
-us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It used a made-up $2,500 zero-rate amount instead of $9,622. It explicitly refused the Idaho QBI deduction and left out the $1,000 dependent care subtraction, giving $140,070. It also subtracted a $410 child credit that the reference does not include."
-us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used an unexplained AGI of $162,405 and a $3,500 head-of-household deduction instead of $24,150, then applied the repealed 5.695% rate. It subtracted a made-up $1,000-per-child Idaho credit, and the offsetting errors happened to land at $7,049."
-us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It made up $1,500-per-dependent deductions and used a 5.305% rate. It never applied the $9,622 zero-rate amount, the $1,000 dependent care subtraction, or the $800 QBI deduction."
-us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used a married-filing-jointly deduction of about $30,000 for a head-of-household filer and arrived at a taxable income of $96,448 that does not follow from its own figures. It then applied the repealed 5.8% rate."
-us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It applied Idaho's old graduated brackets (1%/3%/5.5%), although since 2023 Idaho has used a single flat rate, 5.3% in 2026, above $9,622 for head of household. It also started from a federal taxable income of $121,620 and subtracted a made-up $2,400 Idaho standard deduction."
-us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It identified the grocery credit and the 3 exemptions correctly but used the superseded $120 per-person amount. For 2026 the amount is $155, so 3 x $155 = $465, not $360."
-us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It asserted that Idaho has no refundable state credits and pointed to the household's high income, missing the refundable grocery credit. That credit has no income limit and pays $155 for each of the 3 members, totaling $465."
-us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It recognized the grocery credit for 3 people but replaced its own $120 figure with an invented $170 per person, giving $510. The 2026 amount is $155 per person, which yields $465."
-us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It called Idaho's grocery credit nonrefundable except in limited cases and zeroed it out at this income. The grocery credit is fully refundable with no income test, worth $155 x 3 = $465."
-us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It treated the grocery credit as nonrefundable or not applicable at this income and answered $0. The grocery credit is refundable regardless of income and pays $155 for each of the 3 members, $465 in total."
-us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,It stated that Idaho's grocery credit is nonrefundable and so found no refundable credits. The grocery credit is Idaho's refundable credit: $155 per member x 3 = $465.
-us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,It conceded that the grocery credit is refundable but then excluded it as 'already reflected elsewhere' and answered $0. The refundable grocery credit is exactly what state_refundable_credits captures: $155 x 3 = $465.
-us,scenario_076,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,It applied the outdated $120 per-person grocery credit to 3 people ($360) instead of the 2026 amount of $155 per person ($465).
-us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted Idaho's $205-per-child child tax credit as refundable ($410). That credit is nonrefundable, and the model omitted the refundable grocery credit of $155 x 3 members = $465."
-us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It reported Idaho's $205-per-child child tax credit as refundable ($410). That credit is nonrefundable, so it belongs in pre-refundable state tax. The only refundable state credit is the grocery credit of $155 x 3 = $465."
-us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly applied the refundable grocery credit to 3 members but used the old $120 amount instead of the 2026 amount of $155, giving $360 instead of $465."
-us,scenario_076,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used $120 per person for the grocery credit, which is the pre-2026 level. The 2026 credit is $155 per person, so 3 people yield $465, not $360."
-us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the superseded $120 per-person grocery credit to 3 members ($360). The 2026 per-person amount is $155, which gives $465."
-us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It assumed income excludes the household from Idaho refundable credits. The grocery credit has no income test and pays $155 for each of the 3 members, $465 in total."
-us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It claimed that no Idaho refundable credit applies at this income level, missing the income-independent refundable grocery credit of $155 per member x 3 = $465."
-us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used the pre-2023 grocery credit of $100 per person ($300). The 2026 amount is $155 per person, so 3 members give $465."
-us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Its bare $0 shows it never applied Idaho's refundable grocery credit. That credit is worth $155 for each of the head and two children, $465 in total."
-us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It concluded that the household qualifies for no refundable Idaho credit, overlooking the grocery credit. Every resident member receives it regardless of income: $155 x 3 = $465."
-us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,It applied the old $120 per-person grocery credit to 3 members ($360) instead of the 2026 amount of $155 per person ($465).
-us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It asserted that PolicyEngine models no Idaho refundable credits. PolicyEngine computes id_refundable_credits from the refundable grocery credit, at $155 x 3 members = $465."
-us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the pre-2023 grocery credit of $100 per household member ($300). The 2026 amount is $155 per member, which yields $465 for 3 members."
-us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,It correctly treated the grocery credit as the only refundable Idaho credit but used the outdated $100 per-person amount. The 2026 amount of $155 per person gives $465.
-us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no Idaho refundable credit in the listed facts, missing that the grocery credit needs only Idaho residency. It pays $155 for each of the 3 members, $465 in total."
-us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,It assumed Idaho's refundable credits have income thresholds that this household exceeds. The grocery credit has no income limit and pays $155 x 3 = $465.
-us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly applied the refundable grocery credit to 3 non-senior members but used the superseded $120 amount. At the 2026 amount of $155, the total is $465."
-us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It stated that no refundable Idaho credit applies, omitting the refundable grocery credit of $155 per member for the head and two children ($465)."
-us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied a $120 per-person grocery credit to 3 members ($360), using the pre-2026 amount instead of the 2026 amount of $155 per person ($465)."
-us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It declared that no state refundable credits apply, missing Idaho's refundable grocery credit of $155 for each of the 3 household members ($465)."
-us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the outdated $100 per-person grocery credit ($300). The 2026 amount is $155 per person, which gives $465 for the head and two dependents."
-us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It treated $100 per resident as the permanent-law grocery credit for 2026. The 2026 amount is $155 per person, so 3 members yield $465, not $300."
-us,scenario_076,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly identified the grocery credit as the sole refundable credit but used $120 per member instead of the 2026 amount of $155, giving $360 instead of $465."
-us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,It claimed that the grocery credit phases out fully at this income level. Idaho's grocery credit has no income phase-out and pays $155 x 3 members = $465.
-us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,It applied the pre-2023 grocery credit of $100 per household member ($300) instead of the 2026 amount of $155 per member ($465).
-us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so no answer was scored. The correct derivation is Idaho's grocery credit of $155 x 3 members = $465."
-us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the grocery credit to the head and two non-senior children at the superseded $120 amount ($360). For 2026 the amount is $155 per person, giving $465."
-us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,It dismissed the grocery credit as not triggered at this income level. The credit applies to every resident member regardless of income and pays $155 x 3 = $465.
-us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly found no aged supplement but used the pre-2023 $100 per-member grocery credit ($300). The 2026 amount is $155 per member, which gives $465."
-us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It looked only for an earned income credit and found no refundable Idaho credit, missing the refundable grocery credit of $155 for each of the 3 members ($465)."
-us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It stated that no Idaho refundable credits apply, omitting the grocery credit. Idaho refunds it at $155 per household member, $465 for this 3-person household."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference: AGI $23,006, a $16,100 standard deduction, taxable income $6,906, and 10% tax of $691. It then submitted 951, a number its own derivation does not support. This is a transcription error on the final value."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, although the reference AGI starts from the full $26,006. It also used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100, which gave $1.70 of tax. It then submitted an unexplained $1,402 that none of its steps produce."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived AGI $23,006, a $16,100 standard deduction, taxable income $6,906 and 10% tax of about $691. It then replaced that with an arbitrary $1,080, citing 'slightly higher taxable income', and nothing in its computation supports the adjustment."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took the pre-OBBBA 2025 standard deduction of $15,000 and indexed it to $15,350 for 2026. The OBBBA-set 2026 figure is $16,100. That overstated taxable income at $7,656 instead of $6,906, giving $766 instead of $691."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly computed taxable income of $6,906 and tax of about $691. It then zeroed the liability by invoking a QBI deduction and other credits that a childless W-2 wage earner with no business income cannot claim. No nonrefundable credit applies, so the $691 stands."
-us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 single standard deduction instead of the 2026 figure of $16,100. That gave taxable income of $7,606 instead of $6,906 and tax of $760.60 instead of $690.62."
-us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $15,000 instead of the 2026 figure of $16,100. That overstated taxable income at $8,006 and tax at $800.60."
-us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used an inflation-indexed $15,350 standard deduction built from the pre-OBBBA $15,000 base, not the 2026 figure of $16,100. That gave taxable income of $7,656 and tax of $765.60."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, which cut AGI to $14,617 instead of $23,006. It also applied TCJA-sunset rules, an $8,300 standard deduction plus a $5,300 personal exemption, instead of the permanent $16,100 standard deduction with no exemption."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the outdated $15,000 standard deduction and reached positive taxable income of $8,006. It then declared the tax fully offset by 'available credits', but a childless single wage earner has no nonrefundable credit, so the correct $16,100 deduction leaves $691 of tax."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated wages, which gave AGI of $14,617 instead of $23,006. It then assumed the TCJA sunset, with about $13,500 of combined standard deduction and personal exemption, instead of the 2026 law's $16,100 standard deduction and no exemption."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $8,389 ESI premium, which gave AGI of $14,617 instead of $23,006. It then applied the TCJA-sunset $8,300 standard deduction plus a $5,200 personal exemption instead of the $16,100 standard deduction in force for 2026."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted $0 on the grounds that the standard deduction and capital-loss limit eliminate taxable income. In fact AGI of $23,006 less the $16,100 standard deduction leaves $6,906 taxable at 10%, which is $691."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its AGI of $23,006 is correct. Its $965.60 implies taxable income of $9,656, which means it subtracted only about $13,350 in deductions (a sunset-style standard deduction plus an exemption) instead of the $16,100 standard deduction."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated wages, which gave AGI of $14,617 instead of $23,006. It then applied a sunset-era standard deduction plus a personal exemption. On top of that, it submitted $67 while its explanation states $102."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 for 2026 instead of $16,100. That gave taxable income of $8,006 and tax of $800.60 instead of $690.62."
-us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,450 instead of the enacted $16,100. That gave taxable income of $7,556 and tax of $755.60."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed $26,006 of wages falls below the standard deduction, but the 2026 single standard deduction is $16,100. AGI of $23,006 after the $3,000 capital-loss deduction therefore leaves $6,906 of taxable income and $691 of tax."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the wage income sits below the standard deduction and produces no tax. In fact $23,006 of AGI less the $16,100 standard deduction leaves $6,906 taxed at 10%, and no nonrefundable credit offsets that $691."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a sunset-era $8,150 standard deduction instead of $16,100, which inflated taxable income to $14,856. It then taxed that income with a sunset 15% bracket above $11,925, instead of the 2026 schedule where all $6,906 of taxable income falls in the 10% bracket."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is zero or negative after the standard deduction and capital-loss offset. In fact $26,006 minus the $3,000 loss limit minus the $16,100 standard deduction leaves $6,906 taxable, which is $691 of tax."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters, an $8,300 standard deduction plus a $5,300 personal exemption, instead of the permanent 2026 standard deduction of $16,100 with no personal exemption. That raised taxable income from $6,906 to $9,406."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and subtracted an $8,300 standard deduction plus a $5,300 personal exemption. Current 2026 law allows a $16,100 standard deduction and no exemption, so taxable income is $6,906, not $9,406."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules, an $8,300 standard deduction and a $5,300 personal exemption, instead of the 2026 standard deduction of $16,100 with personal exemptions still zeroed. That produced $9,406 of taxable income instead of $6,906."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, which gave AGI of $14,617 instead of $23,006. It also used the sunset-era $8,300 standard deduction instead of $16,100, and the two errors partly offset to give $632."
-us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so no answer was available to score against the $690.62 reference."
-us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It deducted the full $7,968 of capital losses against wages without the $3,000 annual limit and used the 2025 $15,750 standard deduction instead of $16,100. It then zeroed the result with nonexistent 'personal exemption credits' instead of taxing $6,906 at 10%."
-us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 OBBBA single standard deduction of $15,750 without the 2026 inflation adjustment to $16,100. That gave taxable income of $7,256 and tax of $725.60."
-us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It correctly netted the capital losses to the $3,000 limit but then asserted there is no regular tax. In fact $23,006 of AGI less the $16,100 standard deduction leaves $6,906 taxed at 10%, and no nonrefundable credit applies."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It reached $139,270 of taxable income, so it missed the $1,000 Idaho child-care subtraction. It then applied 5.3% to the full amount with no HoH zero-rate band of about $9,622. It also subtracted a $410 child tax credit that the 2026 computation does not apply, left out the $10 permanent building fund tax, and finally pushed the result up to $7,099 for no stated reason."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It took the $1,000 child-care subtraction but left out the $800 QBI deduction, giving $139,070 instead of $138,270. It then applied 5.3% to the whole amount instead of only the income above the roughly $9,622 HoH zero-rate band. It also subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted non-taxable child support as income and used the single $15,000 standard deduction instead of the $24,150 HoH deduction. It also added an invented $9,380 adjustment and used made-up graduated brackets instead of Idaho's flat 5.3% rate above the zero-rate band. Together these push taxable income to $161,720 and the tax to $9,400."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It gave a rough $5,800 from a 5.3% approximation minus 'dependent credits', with no computation shown. The correct steps are 5.3% × ($138,270 − ~$9,622) = $6,818.34, with no Idaho nonrefundable credit, plus the $10 permanent building fund tax. Its figure is consistent with subtracting credits and deductions that do not apply."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It used the repealed 5.695% rate instead of the 2026 rate of 5.3%, and the 2025 standard deduction of $23,625. It left out the $800 QBI deduction and the $1,000 child-care subtraction. It then taxed nearly the full $140,595 and raised the result to $8,204 with no supporting arithmetic."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used the outdated 5.695% rate and an understated $22,500 HoH standard deduction instead of $24,150. It skipped the QBI and child-care subtractions and used a $3,420 zero band instead of about $9,622. It then inflated its own $7,876 figure to $9,112 without a basis."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It applied 5.3% to about $139,720 with no HoH zero-rate band of about $9,622. It used a $24,500 standard deduction instead of $24,150 and left out the $800 QBI deduction and $1,000 child-care subtraction. It then raised the result above its own $7,405 figure instead of computing $6,818.34 + $10."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It used taxable income of $140,070, leaving out the $800 QBI deduction and the $1,000 child-care subtraction that bring it to $138,270. It used the unindexed $9,346 zero band instead of the 2026 HoH figure of about $9,622. It also subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It filed as single with a $15,000 standard deduction instead of head of household with $24,150. It also deducted three $4,714 personal exemptions, which Idaho no longer allows, and used the repealed 5.695% rate instead of 5.3% above the zero-rate band. On top of that it subtracted a $100 child-care credit."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It used the outdated 5.695% rate on nearly all income and a $16,000 standard deduction instead of the $24,150 HoH deduction. It skipped the QBI and child-care subtractions and the zero-rate band. It then inflated its own $8,443 figure to $9,400 with invented adjustments."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied 5.3% to all of $140,070, skipping the HoH zero-rate band of about $9,622, the $800 QBI deduction and the $1,000 child-care subtraction. It then subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It used the 2022 rate of 5.8% instead of the 2026 rate of 5.3%, and a $19,900 HoH standard deduction instead of $24,150. It also left out the QBI and child-care subtractions, the zero-rate band of about $9,622, and the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It cut AGI to $157,740 by subtracting the $6,480 employer insurance premiums from wages that are already gross, when AGI is $164,220. It used a $23,200 standard deduction instead of $24,150 and the outdated 5.8% rate instead of 5.3% above the zero-rate band. It also left out the QBI and child-care subtractions."
+us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted the $6,480 insurance premiums to get a $157,740 AGI instead of $164,220. It used a $12,155 HoH standard deduction instead of $24,150 and applied the repealed 5.695% rate to the full amount with no zero-rate band. It also submitted $8,247.54, which does not match its own $8,291.07."
+us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a wrong 5.0% flat rate on $140,070 instead of 5.3% on income above the roughly $9,622 HoH zero band. It skipped the $800 QBI and $1,000 child-care subtractions. It also subtracted $450 of child tax credit and CDCC that the Idaho 2026 computation does not apply, and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used an understated $130,867 taxable income instead of $138,270 and the outdated 5.8% rate instead of 5.3%. It skipped the zero-rate band, subtracted a $410 child tax credit that is not applied, and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation. $8,100 matches applying an outdated 5.695–5.8% rate to roughly $140,000 with no zero-rate band. The correct result is 5.3% × ($138,270 − ~$9,622) + $10 = $6,828.34."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It used the repealed 5.695% rate instead of 5.3%, applied it with no zero-rate band, and used an understated taxable income of $129,750 instead of $138,270. It also left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It used the outdated 5.8% rate instead of the 2026 rate of 5.3%, ignored the HoH zero-rate band of about $9,622, and subtracted child tax credits that the 2026 computation does not apply. This overstates the tax at $8,043."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. $7,322 is consistent with applying 5.3% to roughly $138,000–$139,000 with no zero-rate band. The correct steps exclude the first ~$9,622 of HoH income from tax, which gives $6,818.34, and then add the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"$6,985 matches 5.3% on about $139,000 of taxable income with no zero-rate band, minus a $410 child tax credit. The correct steps exempt the first ~$9,622 of HoH income, apply no nonrefundable credit, and add the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation. $7,665 overstates the tax and is consistent with a flat rate above 5.3% on nearly all taxable income with no zero-rate band of about $9,622. The correct result is 5.3% × $128,648 + $10 = $6,828.34."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It wrongly claimed Idaho income tax is not modeled and returned $0. The household owes Idaho tax of 5.3% on taxable income of $138,270 above the HoH zero-rate band, plus the $10 permanent building fund tax, for $6,828.34."
+us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It used the repealed 5.695% rate on all of $141,720 with no zero-rate band and an understated $22,500 standard deduction instead of $24,150. It also left out the $800 QBI deduction and $1,000 child-care subtraction."
+us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It started from federal taxable income, which already reflects the standard deduction, and then subtracted a $23,625 Idaho standard deduction again. That takes taxable income down to $116,170 instead of $138,270. It also subtracted a $410 child tax credit that is not applied and ignored the zero-rate band."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave no computation. $6,494 understates the result: 5.3% on the $128,648 above the HoH zero-rate band gives $6,818.34, plus the $10 permanent building fund tax. Its figure is consistent with subtracting credits or extra deductions that do not apply."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It referred to itemized medical deductions, which do not apply because the $24,150 standard deduction is used. $7,450 is consistent with taxing roughly $140,000 at 5.3% with no zero-rate band of about $9,622."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It applied 5.3% to all of $140,870 using a $23,350 standard deduction instead of $24,150. It skipped the zero-rate band and the QBI and child-care subtractions, then subtracted $346 of child credits that are not applied. It also left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed $132,020 at 5.3% with no zero-rate band, which understates taxable income relative to $138,270. It then subtracted a $440 child credit that the 2026 computation does not apply and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used the single-filer zero-rate band of about $4,930 instead of the head-of-household band of about $9,622. It taxed $139,270, leaving out the $1,000 child-care subtraction. It also subtracted $410 of child tax credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270, skipping the HoH zero-rate band of about $9,622 and the $1,000 Idaho child-care subtraction. It also left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It subtracted the $6,480 employer insurance premiums as an extra Idaho deduction, getting $131,790 instead of $138,270. It used a $9,960 zero band instead of about $9,622. It also subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed all of $139,270 at 5.3% with no HoH zero-rate band and left out the $1,000 child-care subtraction. It then subtracted $410 of Idaho child credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It applied 5.3% to the full $139,270, skipping the roughly $9,622 HoH zero-rate band and the $1,000 child-care subtraction. It then subtracted $410 of child tax credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It got the correct $138,270 taxable income but used the single-filer zero band of $4,980 instead of the HoH band of about $9,622, which overstates the tax by about $245. It then subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It applied a guessed 4% effective rate instead of computing 5.3% on taxable income of $138,270 above the roughly $9,622 HoH zero-rate band and adding the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It used the repealed 5.695% rate on all of $141,157 and explicitly refused the QBI deduction, which Idaho allows. It also skipped the child-care subtraction and the zero-rate band, and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $12,450 HoH standard deduction instead of $24,150 and applied the outdated 5.695% rate to all of $151,770. It skipped the QBI and child-care subtractions and the zero-rate band."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to the full $136,070 instead of 5.3% on taxable income of $138,270 above the roughly $9,622 HoH zero band. It also left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It subtracted the $6,480 insurance premiums to get a $157,740 AGI and used a $12,585 standard deduction instead of the $24,150 HoH deduction. It then applied the outdated 5.8% rate to all of $145,155 with no zero-rate band."
+us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to roughly $135,000–$140,000 as a flat tax with no zero-rate band. Idaho exempts the first ~$9,622 of HoH taxable income, which gives $6,818.34, and then adds the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It explicitly refused the $800 QBI deduction and left out the $1,000 Idaho child-care subtraction, giving $140,070 instead of $138,270. It used a $2,500 zero band instead of about $9,622. It also subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used an unexplained $162,405 AGI instead of $164,220, a $3,500 HoH standard deduction instead of $24,150, and the repealed 5.695% rate. It then subtracted an invented $1,000-per-child Idaho credit. The errors offset each other only partly."
+us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It deducted $3,000 of nonexistent $1,500 dependent deductions and skipped the $800 QBI and $1,000 child-care subtractions. It then applied a 5.305% rate to all of $137,070 with no zero-rate band and left out the $10 permanent building fund tax."
+us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used a married-filing-jointly standard deduction for a head-of-household filer. It made an arithmetic error that cut taxable income to $96,448 and applied the outdated 5.8% rate instead of 5.3% above the HoH zero band."
+us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It used Idaho's repealed graduated brackets of 1%, 3% and 5.5%, an invented $2,400 Idaho standard deduction on top of federal taxable income, and an understated $121,620 federal taxable income. The 2026 law applies a flat 5.3% to taxable income of $138,270 above the roughly $9,622 zero band."
+us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly picked the refundable Idaho grocery credit and counted 3 exemptions. However, it used the old $120 per-person amount instead of the $155 per-person amount for non-seniors in 2026, so it got $360 instead of $465."
+us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model wrongly said Idaho has no refundable individual income tax credits. It missed the refundable grocery credit, which has no income limit and pays $155 for each of the 3 household members, for $465 total."
+us,scenario_076,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model missed Idaho's refundable grocery credit. That credit applies at any income level and pays $155 each for the head and two children, so the correct total is $465, not $0."
+us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the grocery credit and the 3-person count but replaced the correct rate with a made-up $170 per person. The 2026 non-senior amount is $155, giving $465 rather than $510."
+us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model wrongly treated Idaho's grocery credit as a nonrefundable credit that is refundable only in limited cases. The grocery credit is fully refundable regardless of income and pays $155 × 3 = $465.
+us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model wrongly called the Idaho grocery credit nonrefundable or unavailable at this income. It is a refundable per-exemption credit with no income phase-out, worth $155 each for 3 people, or $465."
+us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly classified Idaho's grocery credit as nonrefundable. The credit is refundable and pays $155 per exemption for the head and two children, for a total of $465."
+us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the grocery credit is refundable but then left it out, claiming it was 'already reflected elsewhere.' The state_refundable_credits output is exactly where the grocery credit is counted: $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit amount. For 2026 the non-senior amount is $155, so 3 people get $465, not $360."
+us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model counted Idaho's $205-per-child child tax credit as refundable, but that credit is nonrefundable. It also missed the refundable grocery credit, which is $155 × 3 people = $465."
+us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated Idaho's nonrefundable $205 child tax credit as refundable and left out the refundable grocery credit. The correct refundable amount is the grocery credit of $155 for each of 3 members, or $465."
+us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the right credit and member count but the superseded $120 per-person rate. With the $155 non-senior amount for 2026, the total is $465, not $360."
+us,scenario_076,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied the old $120 per-person grocery credit. For 2026 the amount is $155 per non-senior exemption, so 3 people get $465."
+us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly treated the grocery credit as refundable for all 3 members but used the outdated $120 amount. The 2026 rate is $155 per person, for $465."
+us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model assumed Idaho's refundable credits are income-limited and returned $0. The grocery credit has no income test and pays $155 for each of 3 exemptions, or $465."
+us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model wrongly concluded that no Idaho refundable credit applies at this income. It missed the grocery credit, which is refundable at any income and pays $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a stale $100 per-person grocery credit amount. For 2026 the non-senior amount is $155, so the total is $465, not $300."
+us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model returned $0 and missed Idaho's refundable grocery credit. That credit gives $155 for each of the head and two children, for $465."
+us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model said the household qualifies for no refundable Idaho credit. The grocery credit applies to every resident exemption without an income limit, giving $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model identified the grocery credit and 3 members but used the old $120 amount. The 2026 per-person amount of $155 gives $465.
+us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,The model wrongly claimed PolicyEngine models no refundable Idaho credits. Idaho's refundable grocery credit applies and equals $155 × 3 = $465.
+us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the long-superseded $100 per-member grocery credit amount. The 2026 non-senior amount is $155, so 3 members get $465."
+us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model applied a $100 per-person grocery credit instead of the 2026 amount of $155, so it got $300 instead of $465."
+us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable Idaho credit and missed the grocery credit. That credit is refundable for every exemption at any income level, giving $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed Idaho's refundable credits have income thresholds this household exceeds. The grocery credit has no income phase-out and pays $155 per exemption, for $465."
+us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly used 3 non-senior members but the outdated $120 grocery credit rate. At the 2026 rate of $155 per person, the total is $465."
+us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model said no refundable Idaho credit applies and left out the grocery credit. That credit pays $155 for each of the 3 household members, for $465."
+us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the superseded $120 per-person grocery credit. The 2026 non-senior amount is $155, giving $465 for 3 members."
+us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model missed Idaho's refundable grocery credit, which applies at any income level and pays $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $100 per-person grocery credit instead of the 2026 amount of $155 per non-senior exemption, so it got $300 instead of $465."
+us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model wrongly treated $100 per resident as permanent law for 2026. Current law sets the non-senior grocery credit at $155, so 3 residents get $465."
+us,scenario_076,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the old $120 grocery credit rate. The 2026 amount is $155 per person, giving $465 for the head and two children."
+us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model wrongly assumed the Idaho grocery credit phases out at higher incomes. The credit has no income phase-out and pays $155 per exemption, or $465 for 3 members."
+us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an outdated $100 per-member grocery credit. The 2026 non-senior amount is $155, so 3 members get $465."
+us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was nothing to score. The correct answer is the Idaho grocery credit of $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly found 3 non-senior exemptions but used the old $120 grocery credit. The 2026 amount of $155 per person gives $465.
+us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model wrongly treated the grocery credit as not triggered at this income. The credit is refundable for every resident exemption with no income test, giving $155 × 3 = $465."
+us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly left out any senior supplement but used a $100 base rate. The 2026 non-senior grocery credit is $155 per member, so the total is $465."
+us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model looked only for income-tested credits such as an earned income credit. It missed Idaho's refundable grocery credit, which pays $155 for each of 3 members regardless of income, for $465."
+us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model said no Idaho refundable credits apply, missing the grocery credit. That credit pays $155 per exemption for the head and two children, for $465."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own reasoning reached the correct result: AGI $23,006, the $16,100 standard deduction, taxable income $6,906, and $691 of tax at 10%. It then submitted $951, which matches nothing in its derivation."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 OBBBA figure of $16,100. It also subtracted the $8,389 ESI premiums from wages that were stated as gross, and then submitted $1,402, a number that matches none of its intermediate calculations."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It left out the $3,000 capital-loss deduction, so its taxable income was about $9,906 instead of $6,906. It then estimated about $1,000 of tax and still submitted $0, even though no nonrefundable credit applies."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly computed AGI of $23,006, a $16,100 standard deduction, taxable income of $6,906, and about $691 of tax at 10%. It then submitted $1,080 for no stated reason, contradicting its own math."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took the pre-OBBBA 2025 standard deduction of $15,000 and inflated it to $15,350, missing the 2026 OBBBA figure of $16,100. That overstated taxable income by $750, giving $7,656 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly derived about $691 of tax on $6,906 of taxable income. It then cited CDCC, QBI, and other adjustments that do not apply to a childless wage earner to cut the tax to $0."
+us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 single standard deduction instead of the 2026 OBBBA amount of $16,100. That overstated taxable income by $700, giving $7,606 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the original 2025 standard deduction of $15,000 instead of the 2026 OBBBA amount of $16,100. As a result, it taxed $8,006 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a pre-OBBBA inflation estimate of $15,350 for the 2026 single standard deduction instead of the enacted $16,100. That left $7,656 of taxable income instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premiums from wages that were stated as gross, which pushed AGI down to $14,617. It then applied TCJA-sunset rules ($8,300 standard deduction plus a $5,300 personal exemption) instead of OBBBA's permanent $16,100 standard deduction with no exemption."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of $16,100 and reached $8,006 of taxable income. It then asserted that credits wiped out the tax, but no nonrefundable credit applies to this childless filer, so the 10% tax should have stood."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It treated the $8,389 ESI premiums as a reduction to stated gross wages, which dropped AGI to $14,617. It also assumed the TCJA sunset (about $13,500 of standard deduction plus exemption), although OBBBA made the $16,100 standard deduction permanent and kept the exemption at zero."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premiums from gross wages, giving AGI of $14,617 instead of $23,006. It also applied TCJA-sunset parameters ($8,300 standard deduction plus a $5,200 exemption) instead of the 2026 OBBBA $16,100 standard deduction with no exemption."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It claimed the standard deduction and capital-loss limit wipe out all tax. In fact, AGI of $23,006 is well above the $16,100 standard deduction, which leaves $6,906 of taxable income and $690.62 of tax at 10%."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It got AGI right at $23,006, but its $965.60 implies $9,656 of taxable income, so it subtracted only about $13,350. That deduction level matches the TCJA-sunset standard deduction plus exemption, not OBBBA's $16,100 standard deduction."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premiums from wages that were stated as gross, which pushed AGI down to $14,617. It then applied a TCJA-sunset standard deduction plus personal exemption instead of the 2026 $16,100 standard deduction, and it submitted $67, which contradicts the $102 in its own explanation."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the original 2025 standard deduction of $15,000 for 2026 instead of the OBBBA-indexed $16,100. As a result, it taxed $8,006 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,450 instead of the enacted OBBBA amount of $16,100. That overstated taxable income by $650, giving $7,556 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It claimed $26,006 in wages falls below the standard deduction. The 2026 single standard deduction is $16,100, so after the $3,000 capital-loss deduction, $6,906 remains taxable at 10%."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It said wage income is low relative to the standard deduction and set the tax to $0. In fact, AGI of $23,006 minus the $16,100 standard deduction leaves $6,906 of taxable income and $690.62 of tax, with no offsetting nonrefundable credits."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used an $8,150 standard deduction with no exemption instead of OBBBA's $16,100, which inflated taxable income to $14,856. It also pushed part of that income into the 12%/15% range instead of keeping all $6,906 in the 10% bracket."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It asserted that taxable income is zero or negative. AGI of $23,006 minus the $16,100 standard deduction leaves $6,906 of positive taxable income, which is taxed at 10%."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters ($8,300 standard deduction plus a $5,300 personal exemption) for 2026. OBBBA made the TCJA structure permanent with a $16,100 single standard deduction and a zero exemption, so it overstated taxable income by $2,500 ($9,406 instead of $6,906)."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,300 standard deduction plus a $5,300 exemption, giving $9,406 of taxable income. Under OBBBA, the 2026 single standard deduction is $16,100 with no exemption, so taxable income is $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules ($8,300 standard deduction plus a $5,300 exemption) instead of OBBBA's permanent $16,100 standard deduction with no exemption. That produced $9,406 of taxable income instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premiums from wages that were stated as gross, giving AGI of $14,617 instead of $23,006. It then applied an $8,300 sunset-era standard deduction instead of the 2026 OBBBA amount of $16,100."
+us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It subtracted the full $7,968 of capital losses from wages, ignoring the $3,000 annual limit on deducting net capital losses against ordinary income. It then used $15,750 instead of the $16,100 standard deduction and cut the tax to $0 with nonexistent personal exemption credits."
+us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 OBBBA single standard deduction of $15,750 instead of the inflation-indexed 2026 amount of $16,100. That overstated taxable income by $350, giving $7,256 instead of $6,906."
+us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It asserted that no regular tax remains after the standard deduction and the $3,000 capital-loss deduction. In fact, $23,006 minus $16,100 leaves $6,906 taxable at 10%, and no nonrefundable credits apply."
us,scenario_077,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly determined that the greater-of-earned-income-or-AGI rule uses $26,006 and fully phases out the childless EITC, then submitted $56 despite its own zero-credit computation. No other refundable credit supplies the submitted amount."
us,scenario_077,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly left $9 of childless EITC after acknowledging $26,006 of wages. EITC phaseout uses the greater of earned income or AGI, so the capital-loss deduction does not prevent the credit from being fully phased out."
us,scenario_077,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model reduced $26,006 of wages by the $8,389 employer-sponsored insurance premium and treated $17,617 as EITC earned income. Employer premiums do not reduce EITC earned income here, so the phaseout must use $26,006 and eliminates the childless credit."
@@ -6155,165 +6317,169 @@ us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals
us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model’s explanation correctly states that the one-person household exceeds the SNAP net-income limit and explicitly concludes “value = 0,” but it submitted 276. The numeric output contradicts its own eligibility conclusion and should have been 0."
us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person SNAP net-income and benefit calculations to its $26,006 of wages. After the allowable earned-income and standard deductions, the household does not generate an eligible allotment, so the asserted $2,400 benefit has no valid computation."
us,scenario_077,snap,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly awarded the $23 minimum allotment after determining that the regular formula produced zero. Broad-based categorical eligibility does not entitle a household failing the applicable net-income or benefit test to the minimum allotment; the minimum applies only to an otherwise eligible one- or two-person household.
-us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly worked out 3% × ($23,006 − $12,500) = $315.18, but it used the unindexed $12,500 deduction instead of the 2026 indexed $12,835. It then replaced that result with an unexplained $496 'after accounting for the deduction structure', and no Louisiana rule supports that figure."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used Louisiana's statutory $12,500 single standard deduction without the inflation indexing that begins in 2026. The correct figure is 3% × ($23,006.18 − $12,835) = $305.14, not 3% × $10,506 = $315.18."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It said Louisiana has no state individual income tax. Louisiana levies a flat 3% income tax on AGI minus a $12,835 standard deduction in 2026, which produces $305.14 here."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It correctly identified the post-reform flat 3% rate and that there is no separate personal credit. However, it subtracted the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which overstated taxable income by $335 and gave $315 instead of $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied the flat 3% rate to $23,006 − $12,500. It missed that Louisiana indexes the $12,500 single deduction for inflation starting in 2026, making it $12,835, so its taxable income of $10,506 should be $10,171.18 and the tax $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It computed about $315 using the unindexed $12,500 deduction, then raised the answer to $446 citing capital-loss treatment and rounding, which no rule supports. The capital loss is already capped at $3,000 inside AGI, and the correct deduction is the indexed $12,835, which gives $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the $12,500 standard deduction from 2025 instead of the 2026 inflation-indexed $12,835. That left $10,506 of taxable income instead of $10,171.18 and $315.18 of tax instead of $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It subtracted a $4,500 personal exemption and an $801 deduction for federal income tax paid from the $23,006 AGI, which left $17,705 taxable at 3% and gave $531. Louisiana ended the federal income tax deduction after 2021, and from 2025 a single standard deduction ($12,835 in 2026) replaced the exemption, leaving $10,171.18 taxable at 3% and $305.14 of tax."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It reached about $315 using the unindexed $12,500 deduction and then raised it to $448 by citing 'federal-based deductions' and 'personal exemption credits'. Neither applies to this household in 2026, and both would lower the tax, not raise it. The correct base is $23,006.18 − $12,835 at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It explicitly assumed a $12,500 standard deduction and ignored Louisiana's inflation indexing from 2026, which makes the single deduction $12,835. That overstated the tax at $315.18 instead of $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its taxable base of $7,606 means it subtracted $15,400 from the $23,006 AGI, which it called the standard deduction without stating the amount. Louisiana's 2026 single standard deduction is $12,835, which leaves $10,171.18 taxable at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. As a result it taxed $10,506 instead of $10,171.18 at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied Louisiana's repealed pre-2025 law: a deduction for federal income tax, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. For 2026 Louisiana taxes AGI minus the $12,835 standard deduction at a flat 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used $12,500 as the single standard deduction and missed the 2026 inflation adjustment to $12,835. The result was $315.18 instead of 3% × $10,171.18 = $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer-sponsored insurance premiums from the stated gross wages, cutting AGI to $14,617 instead of $23,006.18. It then applied the repealed $4,500 exemption and the old 1.85% bottom bracket instead of the $12,835 deduction and the flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no derivation. $158 matches the repealed 1.85% bottom-bracket rate on about $8,500 of income. The correct calculation is the flat 3% rate on $23,006.18 − $12,835 = $10,171.18, which gives $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It removed the $8,389 ESI premiums from gross wages, giving an AGI of $14,617. It then subtracted both a $4,500 standard deduction and a $4,500 personal exemption and taxed the rest at the repealed 1.85% rate. The 2026 law is a single $12,835 deduction from the $23,006.18 AGI and a flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $14,617 by subtracting the ESI premiums from the stated gross wages. It then used the repealed $4,500 combined exemption and the 1.85% bracket, instead of the $12,835 standard deduction and flat 3% rate applied to the $23,006.18 AGI."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It said deductions and exemptions wipe out the tax. Louisiana's only 2026 deduction is the $12,835 standard deduction, which leaves $10,171.18 of taxable income and $305.14 of tax at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $315.18 comes from subtracting the unindexed $12,500 deduction from $23,006. The 2026 inflation-adjusted Louisiana standard deduction is $12,835, which gives $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It subtracted a separate standard deduction and personal exemption, which is the pre-2025 structure. Its explanation also states $130 while the submitted value is $152, and both are well below the correct 3% × ($23,006.18 − $12,835) = $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It claimed PolicyEngine does not model Louisiana income tax and returned $0. The engine applies Louisiana's flat 3% tax to AGI minus the $12,835 standard deduction, which gives $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly capped the capital loss at $3,000 and used the flat 3% rate. However, it used the unindexed $12,500 deduction instead of the 2026 inflation-adjusted $12,835, which overstated taxable income by $335."
-us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the repealed 1.85% bottom-bracket rate instead of Louisiana's flat 3% rate. It also used the unindexed $12,500 deduction instead of $12,835."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed taxable income after the standard deduction is below a threshold. AGI of $23,006.18 minus the $12,835 deduction leaves $10,171.18, and Louisiana's flat 3% rate has no zero bracket, so the tax is $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed zero tax because wages are low and never applied the Louisiana calculation. AGI of $23,006.18 exceeds the $12,835 standard deduction by $10,171.18, which is taxed at 3% for $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without Louisiana's 2026 inflation indexing, which makes it $12,835. That gave $315.18 instead of $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 deduction instead of the 2026 indexed $12,835. That left $10,506 instead of $10,171.18 taxable at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the 2025 statutory $12,500 deduction to 2026 and missed the inflation adjustment to $12,835. This overstated the tax by about $10."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It subtracted the repealed $4,500 personal exemption on top of the $12,500 standard deduction. Louisiana's 2025 reform replaced the exemption with a single standard deduction, which is $12,835 in 2026, so taxable income is $10,171.18, not $6,006."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It subtracted a $12,875 standard deduction, which it called inflation-adjusted, from a $23,006 AGI and taxed the remaining $10,131 at 3%, for $303.93. $12,875 is the provisional amount in the Department of Revenue's 2026 withholding tables and estimated-tax worksheet; the reference's $12,835 leaves $10,171.18 of the engine's $23,006.18 AGI taxable and $305.14 of tax. The $40 larger deduction accounts for $1.20 of the $1.21 gap. The rest comes from the $0.18 of wages that the prompt's $26,006 leaves out and from rounding to the cent; with $12,875 the engine gives $303.94."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction. The 2026 Louisiana single deduction is $12,835, which gives 3% × $10,171.18 = $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It applied $12,500 as the single-filer deduction and missed the 2026 inflation indexing to $12,835. The result was $315.18 instead of $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used Louisiana's $12,500 base-year standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated taxable income by $335 and the tax by about $10."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed deductions and exemptions reduce taxable income to zero. The only 2026 Louisiana deduction is the $12,835 standard deduction, which leaves $10,171.18 of the $23,006.18 AGI taxable at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It applied the repealed pre-2025 law: a $4,500 exemption and graduated 1.85%/3.5% brackets. For 2026 Louisiana uses a $12,835 standard deduction and a flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. It then rounded 3% × $10,506 to $315 instead of computing 3% × $10,171.18 = $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It correctly noted that the federal-tax deduction was repealed and used the flat 3% rate. However, it used the $12,500 deduction without the 2026 inflation adjustment to $12,835."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premiums from gross wages, giving an AGI of $14,617 instead of $23,006.18. It then used the repealed $4,500 deduction and the 1.85% bracket instead of the $12,835 standard deduction and the flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the flat 3% rate after the unindexed $12,500 deduction. The 2026 single deduction is $12,835, which leaves $10,171.18 taxable and $305.14 of tax."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so no substantive answer exists to grade against the $305.14 from 3% × ($23,006.18 − $12,835)."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835, so it taxed $10,506 instead of $10,171.18 at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said Louisiana has no state income tax. Louisiana imposes a flat 3% individual income tax on AGI minus the $12,835 standard deduction, which gives $305.14 here."
-us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction plus a repealed $4,500 Louisiana personal exemption. Louisiana does not use the federal standard deduction; it allows only its own $12,835 standard deduction in 2026, which leaves $10,171.18 taxable at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It deducted the full $7,968 net capital loss instead of capping it at $3,000, which understated AGI at $18,038. It then applied the repealed $4,500 exemption and graduated 1.85%/3.5% brackets instead of the $12,835 standard deduction and flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $347.24 implies about $11,575 of taxable income at 3%, which means a deduction of only about $11,430 from the $23,006.18 AGI. The correct 2026 Louisiana single standard deduction is $12,835, which leaves $10,171.18 taxable and $305.14 of tax."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction instead of the 2026 inflation-adjusted $12,835 and got $315.18. It then replaced that with $496, citing a 'deduction structure' it never identified. The correct figure is 3% of ($23,006 − $12,835), which is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"The model applied the 2025 single standard deduction of $12,500. Louisiana indexes that deduction for inflation from 2026, so it is $12,835. That leaves $10,171 taxable, not $10,506, and tax of $305.14 rather than $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no state income tax. Louisiana taxes individual income at a flat 3%, and here that applies to $10,171 of taxable income after the $12,835 standard deduction, giving $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed exemptions and deductions brought the taxable base to zero. Louisiana's only deduction for a single filer is the $12,835 indexed standard deduction, which leaves $10,171 of the $23,006 AGI taxable at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"The model used the flat 3% rate correctly but took the $12,500 standard deduction without the 2026 inflation indexing that raises it to $12,835. That overstated taxable income by $335, giving $315 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 from $23,006 AGI. The 2026 Louisiana single standard deduction is the inflation-indexed $12,835, so the correct tax is 3% × $10,171 = $305.14, not $315."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"The model first reached about $315 using the unindexed $12,500 deduction. It then raised the answer to $446, citing 'capital-loss treatment and rounding', which has no basis. Federal AGI already reflects the $3,000 loss cap, and with the indexed $12,835 deduction the tax is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the 2025 $12,500 standard deduction rather than the 2026 inflation-adjusted $12,835. That gave $10,506 of taxable income and $315.18 of tax, instead of $10,171 and $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"The model combined the new 3% rate with the old Louisiana structure: a $4,500 personal exemption and a deduction for federal income tax paid. The 2025 reform repealed both and replaced them with a single standard deduction, $12,835 in 2026. Its taxable income of $17,705 should have been $10,171."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"The model computed about $315 with the unindexed $12,500 deduction, then added unexplained 'federal-based deductions' and 'personal exemption credits' to reach $448. Louisiana has no such add-ons after the reform. With the indexed $12,835 deduction the tax is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed a $12,500 standard deduction and missed the 2026 inflation indexing to $12,835. Its tax came out at $315.18 instead of 3% × $10,171.18 = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"The model's $7,606 taxable base means it subtracted a $15,400 deduction from $23,006 AGI. That is close to a federal standard deduction, not Louisiana's. The Louisiana single standard deduction for 2026 is $12,835, which leaves $10,171 taxable and $305.14 of tax."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction. Louisiana's 2026 single standard deduction is $12,835 after inflation adjustment, so taxable income is $10,171 and tax is $305.14, not $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 Louisiana law: a deduction for federal income tax paid, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. For 2026 the law is a flat 3% on AGI minus the $12,835 indexed standard deduction, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 instead of the 2026 inflation-indexed Louisiana standard deduction of $12,835. That overstated taxable income by $335 and gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $8,389 ESI premiums from wages a second time, reaching a $14,617 AGI; the correct AGI is $23,006. It also used the repealed $4,500 exemption and the old 1.85% bottom bracket. Under 2026 law, 3% × ($23,006 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation. The correct calculation is $23,006 AGI minus the $12,835 indexed standard deduction, giving $10,171 taxed at 3%, or $305.14. Its $158 implies only about $5,300 taxable at 3%, so it overstated deductions or exemptions under the old pre-reform structure."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the ESI premiums from wages again, reaching a $14,617 AGI instead of $23,006. It stacked a $4,500 standard deduction and a $4,500 personal exemption, both repealed, and used the old 1.85% bracket. Under 2026 law the tax is a flat 3% on $23,006 − $12,835, which is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"The model cut AGI to $14,617 by subtracting ESI premiums that are not deducted from the reported gross wages, so AGI should be $23,006. It then applied the repealed $4,500 exemption and the 1.85% bracket instead of the $12,835 standard deduction and the flat 3% rate, which give $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions eliminate the tax. The only deduction is the $12,835 single standard deduction, which leaves $10,171 of the $23,006 AGI taxable at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"The model's $315.18 matches 3% of $23,006 minus the 2025 deduction of $12,500. It missed that Louisiana indexed the deduction to $12,835 for 2026, which makes the tax $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"The model relied on a pre-reform 'standard deduction and personal exemption' structure, and its explanation states $130 while it submitted $152. Both figures fall short of the correct 3% × ($23,006 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"The model asserted that PolicyEngine does not model Louisiana income tax and answered $0. Louisiana's flat 3% tax applies to the $10,171 left after the $12,835 standard deduction, giving $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 single standard deduction rather than the 2026 inflation-adjusted $12,835. Taxable income should be $10,171, not $10,506, and tax $305.14, not $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"The model taxed $10,506 at the repealed 1.85% bottom bracket instead of Louisiana's flat 3%. It also used the unindexed $12,500 deduction rather than $12,835. The correct tax is 3% × $10,171 = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"The model claimed taxable income after the deduction and exemption falls below a bracket threshold. Louisiana has no zero bracket and no separate exemption. Taxable income is $23,006 − $12,835 = $10,171, taxed at 3% for $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"The model assumed low wages produce no tax and skipped the calculation. $23,006 AGI minus the $12,835 standard deduction leaves $10,171 taxable at 3%, which is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the 2025 $12,500 single standard deduction and missed its 2026 inflation indexing to $12,835. Its tax was $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"The model handled the $3,000 capital-loss cap correctly but used the unindexed $12,500 deduction instead of the 2026 amount of $12,835. That overstated taxable income by $335 and tax by $10.04."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 rather than the inflation-indexed 2026 Louisiana standard deduction of $12,835. That gave $10,506 taxable and $315.18 of tax instead of $10,171.18 and $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"The model stacked a $4,500 personal exemption on top of the standard deduction. The 2025 reform repealed the personal exemption and folded it into the standard deduction, which is $12,835 for 2026. Taxable income is $10,171, not $6,006, so tax is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"The model correctly indexed the $12,500 deduction for 2026 but overshot the adjustment, using $12,875 instead of $12,835. That understated taxable income by $40 and gave $303.93 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction. Louisiana's 2026 single standard deduction is $12,835, so the tax is 3% × $10,171.18 = $305.14, not $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"The model applied the 2025 $12,500 single-filer deduction and missed the 2026 inflation indexing to $12,835. Its result was $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction rather than the 2026 indexed $12,835. That overstated taxable income by $335 and gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions bring taxable income to zero. The $12,835 Louisiana standard deduction is well below the $23,006 AGI, which leaves $10,171 taxable at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 law: a $4,500 exemption and graduated 1.85%/3.5% brackets. The 2026 law is a $12,835 standard deduction and a flat 3% rate, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"The model used the flat 3% rate but the unindexed $12,500 deduction. With the 2026 inflation-adjusted $12,835 deduction, taxable income is $10,171 and tax is $305.14, not $315."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"The model correctly noted that the federal-tax deduction was repealed but used the 2025 $12,500 standard deduction instead of the 2026 indexed $12,835. Its tax was $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $8,389 ESI premiums from the reported gross wages, giving a $14,617 AGI instead of $23,006. It also used the repealed $4,500 deduction and the old 1.85% bracket. The correct tax is 3% × ($23,006 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 single standard deduction for 2026. Louisiana indexes that deduction to $12,835 for 2026, which makes taxable income $10,171 and tax $305.14 rather than $315."
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $305.14 reference."
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"The model applied the unindexed $12,500 deduction rather than the 2026 inflation-adjusted $12,835. That gave $315.18 instead of 3% × $10,171.18 = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no state income tax. Louisiana levies a flat 3% individual income tax, which here applies to $23,006 AGI minus the $12,835 standard deduction, giving $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the federal standard deduction of $16,100 plus a repealed $4,500 Louisiana exemption. Louisiana allows only its own $12,835 single standard deduction for 2026. Taxable income is $10,171, not $2,406, and tax is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"The model did not cap the net capital loss at $3,000, so its AGI was $18,038 instead of $23,006. It then used the repealed $4,500 exemption and graduated 1.85%/3.5% brackets instead of the $12,835 standard deduction and the flat 3% rate, which give $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation. At 3%, its $347.24 implies about $11,575 of taxable income, which means it subtracted roughly $11,431 from the $23,006 AGI. The correct deduction is the 2026 indexed $12,835, which leaves $10,171 taxable and $305.14 of tax."
us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero."
us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero."
us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25."
us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract."
us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax. Those taxes push SALT to the $40,400 cap, which lowers taxable income to $131,526.91. It then mis-added its own bracket sum: $1,240 + $4,560 + $12,166 + $9,544 is $27,510, but it wrote $33,510 and submitted $33,499."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT figure was real estate tax plus about $6,905 of Maryland state income tax, or $33,365, and it left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap. Its $61,575 of itemized deductions is short of the correct $68,610, so it taxed $138,562 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $2,606 of unreimbursed employee business expenses from AGI. Under the permanent TCJA/OBBBA rules these are suspended miscellaneous itemized deductions, not an above-the-line adjustment. It also limited SALT to real estate tax, dropping state and county income tax that brings SALT to the $40,400 cap. Finally, it used outdated 2024 figures (a $14,600 standard deduction) instead of the 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the OBBBA 2026 cap of $40,400, which cut $30,400 from itemized deductions and gave taxable income of $161,927 instead of $131,526.91. It also used $107,450 as the top of the 22% bracket; the 2026 figure is $105,700."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired $10,000 SALT cap instead of the 2026 $40,400 cap and computed $31,424. It then discarded that figure without explanation and submitted $36,757, which matches the tax under the $16,100 standard deduction ($36,766.86), not itemizing."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions by $30,400 and put taxable income at $161,927 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It set SALT at $33,396, which is real estate tax plus $6,936 of Maryland state income tax, and left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap. That makes itemized deductions $68,610 and taxable income $131,526.91, not $138,531."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the pre-OBBBA $10,000 SALT cap instead of $40,400, which gave taxable income of $161,927. It also made up inflation-adjusted brackets (for example, 22% up to $106,150) instead of using the 2026 schedule, where the 22% bracket ends at $105,700."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of $40,400 and left the $1,631.91 state tax refund out of income, arriving at about $31,069. It then raised the answer to $42,800 by citing Medicare surtax interactions, but payroll Medicare taxes are not part of federal income tax before refundable credits."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT was real estate tax plus about $7,000 of Maryland state income tax, and it left out Maryland county income tax. Including county tax takes SALT to the $40,400 cap, so itemized deductions are $68,610, not about $61,700."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the old $10,000 cap instead of the 2026 cap of $40,400, which gave taxable income of $161,927 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026, so it applied a $5,063 personal exemption, the pre-TCJA 10/15/25/28% brackets and an uncapped SALT deduction. OBBBA made the TCJA brackets permanent, kept the personal exemption at zero and set a $40,400 SALT cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law in 2026, with a $5,300 personal exemption and pre-TCJA single brackets. OBBBA made the TCJA brackets permanent with no personal exemption, and SALT is capped at $40,400, not uncapped."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap. As a result it taxed $145,467 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as a post-TCJA-sunset year and applied a $5,150 personal exemption, uncapped SALT and the 10/15/25/28% brackets. OBBBA kept the TCJA brackets, repealed the exemption and capped SALT at $40,400. It also left the $1,631.91 taxable state refund out of income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing. Itemized deductions total $68,610 ($40,400 capped SALT plus $28,210 mortgage interest), far more than the standard deduction. It also left the $1,631.91 state refund out of income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: restored personal exemptions, no SALT cap, and an AMT triggered by the uncapped SALT. OBBBA made the TCJA brackets and zero exemption permanent with a $40,400 SALT cap. Under those rules, regular tax on $131,526.91 is $24,164.46, and AMT does not apply."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset, deducting a $5,150 personal exemption and using pre-TCJA bracket rates. OBBBA made the TCJA single brackets permanent with no personal exemption. It also left out Maryland county income tax, which takes SALT to the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction instead of itemizing $68,610 ($40,400 capped SALT plus $28,210 mortgage interest). That left taxable income far above the correct $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,050 personal exemption, uncapped SALT and pre-TCJA brackets. OBBBA made the TCJA 10/12/22/24% brackets permanent and repealed the personal exemption. SALT is limited to the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 matches about $161,900 of taxable income, which is what the old $10,000 SALT cap produces. The correct SALT deduction is the 2026 OBBBA cap of $40,400, which gives taxable income of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-expiration rules to 2026, even though OBBBA made the TCJA brackets and zero personal exemption permanent. It also itemized only mortgage interest and real estate tax, leaving out the Maryland state and county income tax that takes SALT to the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing the $68,610, and it left the state refund out of income. It then subtracted a made-up $12,000 child and dependent care credit, even though the household has no child and no care expenses."
-us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the expired $10,000 limit instead of the 2026 cap of $40,400, which gave taxable income of $161,927 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It stated a $10,000 SALT cap instead of the 2026 cap of $40,400. Its $42,034 is higher than the tax even under the standard deduction ($36,766.86), so it also misapplied the 2026 single brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It then overstated the tax on even that base, since 2026 brackets on $145,467 give $27,510.08."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT was only the $26,460 real estate tax, leaving out Maryland state and county income tax that bring SALT to the $40,400 cap. It then miscomputed the brackets: 2026 single rates on its own $145,467 base give $27,510.08, not $29,576.96."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 corresponds to taxable income of about $138,956, so its SALT was roughly $33,000: real estate tax plus Maryland state income tax only. It left out Maryland county income tax, which pushes SALT to the $40,400 cap and taxable income down to $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($54,670) and did not treat Maryland state and county income tax as deductible SALT. Those taxes bring SALT to the $40,400 cap and itemized deductions to $68,610."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the $26,460 real estate tax and left out Maryland state and county income tax, which take SALT to the $40,400 cap. That overstated taxable income by $13,940."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It stated that no other deductible tax payments were reported. It overlooked that the household's own Maryland state and county income tax liability counts toward the SALT deduction. That liability brings SALT to the $40,400 cap, so itemized deductions are $68,610, not $54,670."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $26,460 of real estate tax as SALT and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap, so taxable income is $131,526.91, not $145,467."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added about $6,936 of Maryland state income tax to real estate tax but left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap, cutting taxable income to $131,526.91 from its $138,530.66."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It included estimated Maryland state income tax in SALT but left out Maryland county income tax, so its SALT stayed near $33,400. With county tax included, SALT hits the $40,400 cap and itemized deductions total $68,610."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no reasoning, and its $41,500 is higher than the tax under the standard deduction ($36,766.86). The correct calculation itemizes $68,610 (the $40,400 SALT cap plus $28,210 mortgage interest) and applies 2026 single brackets to $131,526.91 for $24,164.46. Its answer is consistent with skipping itemization and also overstating the bracket tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,427 personal exemption and 2017 brackets indexed forward. OBBBA made the TCJA 10/12/22/24% brackets permanent with no exemption. It also limited SALT to real estate tax, leaving out the Maryland state and county income tax that takes SALT to the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It restored the 10/15/25/28% brackets and a $5,300 personal exemption as if TCJA had sunset. OBBBA made the TCJA brackets permanent with no personal exemption. It also left out Maryland county income tax, so its SALT stayed below the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It also used lower 10% and 12% bracket thresholds than the 2026 schedule ($12,400 and $50,400)."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax under pre-TCJA brackets with a $5,434 personal exemption and uncapped SALT. OBBBA made the TCJA single brackets permanent, eliminated the exemption and capped SALT at $40,400."
-us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $26,460 real estate tax as the whole SALT deduction and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap, so taxable income is $131,526.91, not $145,467."
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits and no reasoning, so no substantive answer exists to score."
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the correct $40,400 cap but counted only real estate tax as SALT, leaving out Maryland state and county income tax. With those included, SALT reaches the cap and itemized deductions are $68,610, not $54,670."
-us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It submitted $0 with no derivation. A single filer with $200,136.91 of AGI and $131,526.91 of taxable income owes $24,164.46 under the 2026 brackets, and no nonrefundable credit offsets it."
-us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It used the correct 2026 SALT cap but counted only the $26,460 real estate tax toward it. It left out Maryland state and county income tax, which take SALT to the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, which gave taxable income of $161,927. It then applied 2024 brackets as a stand-in for 2026."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 32% rate below its 2026 threshold of $201,775. It then invented an AMT computation with no exemption and a made-up foreign-tax offset. With the $90,100 AMT exemption, tentative minimum tax is far below the regular tax of $24,164.46 on $131,526.91, so AMT does not apply."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $26,460 of real estate taxes as SALT and left out the Maryland state and county income tax, which pushes SALT to the $40,400 cap. That understated itemized deductions at $54,670 instead of $68,610. It also made an arithmetic error: its own bracket sum was $27,510, but it reported about $33,499."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Put Maryland income tax at about $6,905, leaving out the county local income tax and understating the state tax, so SALT came to $33,365 instead of the $40,400 cap. Taxable income came out about $7,000 too high ($138,562 instead of $131,527)."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly subtracted $2,606 of unreimbursed employee business expenses above the line, even though they are not deductible. It also left Maryland state and local income tax out of SALT, so SALT was $26,460 instead of the $40,400 cap. Its bracket math does not match even its own taxable income of $142,861."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Never computed the tax and entered $0, even though its own reasoning said taxable income was well above $100k. The correct calculation (itemized deductions of $68,610, taxable income of $131,526.91, 2026 brackets) gives $24,164.46."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the pre-OBBBA $10,000 SALT cap. For 2026 the cap is $40,400, and real estate taxes plus Maryland income tax reach that cap. As a result, itemized deductions were understated by $30,400 and taxable income was overstated at $161,927."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the old $10,000 instead of the 2026 $40,400 cap that real estate taxes plus Maryland income tax reach. It then abandoned its own $31,424 calculation and reported an unsupported $36,757."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Used the expired $10,000 SALT cap instead of the 2026 OBBBA cap of $40,400. That cut itemized deductions to $38,210 instead of $68,610 and left taxable income about $30,400 too high."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly used the $40,400 cap but put Maryland income tax at only $6,936, leaving out the county local income tax, so SALT was $33,396 instead of the capped $40,400. Taxable income came out $138,531 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the $10,000 SALT cap instead of the 2026 $40,400 cap, which overstated taxable income at $161,927. It also used made-up inflation-adjusted brackets instead of the actual 2026 thresholds."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the old $10,000 SALT cap instead of $40,400 and left the $1,632 refund out of AGI. It then raised its own $31,069 result to $42,800 by wrongly folding in Medicare surtax, which is not part of income tax before credits."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Estimated Maryland income tax at about $7,000, leaving out the county local income tax, so SALT of about $33,460 fell short of the $40,400 cap that applies. Taxable income came out about $138,445 instead of $131,527."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions at $38,210 and overstated taxable income at $161,927."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired in 2026 and applied a personal exemption and the pre-TCJA 10/15/25/28% brackets. OBBBA made the TCJA brackets permanent, with no personal exemption and a $40,400 SALT cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset law: a $5,300 personal exemption, uncapped SALT, and pre-TCJA brackets. Under OBBBA, 2026 has no personal exemption, SALT is capped at $40,400, and the 10/12/22/24% brackets apply."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only real estate taxes as SALT and left out Maryland state and local income tax, which pushes SALT to the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired: it used uncapped SALT, a $5,150 personal exemption, and pre-TCJA 15/25/28% brackets. OBBBA keeps the TCJA brackets with no exemption and a $40,400 SALT cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction instead of itemizing $68,610 (SALT capped at $40,400 plus $28,210 of mortgage interest). It also left the $1,632 refund out of income, which overstated taxable income at $183,505."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired, which brings back personal exemptions and uncapped SALT, and then added AMT. Under OBBBA the TCJA brackets continue, SALT is capped at $40,400, and no AMT applies because tentative minimum tax is below the regular tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a $5,150 personal exemption and pre-TCJA brackets, with only about $6,886 of Maryland income tax counted. Current 2026 law has no exemption, the 10/12/22/24% brackets, and SALT capped at $40,400."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Used the standard deduction instead of itemizing $68,610 of SALT and mortgage interest. That roughly doubled the taxable income attributable to deductions and inflated the tax to $33,020."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Treated 2026 as post-TCJA-sunset, with uncapped SALT of $37,999, a $5,050 personal exemption, and pre-TCJA rates. Under OBBBA, SALT is capped at $40,400, there is no exemption, and the TCJA brackets apply."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 answer implies taxable income of about $161k, which matches capping SALT at the old $10,000. The correct figure uses the 2026 $40,400 cap, for itemized deductions of $68,610 and taxable income of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-expiration rules (personal exemption and pre-TCJA brackets) instead of OBBBA's permanent TCJA brackets. It also left Maryland income tax out of SALT, which should be capped at $40,400."
+us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction instead of itemizing $68,610. It also subtracted a made-up $12,000 child and dependent care credit for a single filer with no children and no care expenses."
+us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 $40,400 cap. That understated itemized deductions at $38,210 instead of $68,610 and overstated taxable income at $161,927."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the outdated $10,000 SALT cap and mixed up the standard deduction with itemizing. Its $42,034 even exceeds the roughly $36,767 tax under the standard deduction, so it also misapplied the brackets."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and real estate taxes, leaving out Maryland state and local income tax, which raises SALT to the $40,400 cap. Taxable income was overstated by about $14,000."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT, so it used $26,460 instead of the $40,400 cap and got taxable income of $145,467. It then misapplied the brackets: $29,576.96 does not match the $27,510.08 that its own taxable income produces."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 implies taxable income of about $138,956, which means it counted only part of the Maryland income tax (no county local tax) in SALT. The full state-plus-county tax plus real estate taxes reach the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to the $26,460 of real estate taxes and left out the Maryland state and county income tax, which is deductible and raises SALT to the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and real estate taxes and left Maryland state and local income tax out of SALT. SALT should be capped at $40,400, so it understated deductions by $13,940."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly concluded that no deductible tax payments existed besides real estate taxes. The Maryland state and county income tax on $198,505 of wages is deductible as SALT and raises SALT to the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT and deducted only $26,460 of real estate taxes instead of the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only about $6,936 of Maryland income tax and left out the county local income tax, so SALT of $33,396 stayed under the $40,400 cap instead of reaching it. Taxable income was overstated by about $7,000."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Underestimated Maryland income tax by leaving out the county local income tax, so SALT stopped at about $33,400 instead of the $40,400 cap. Taxable income came out about $138,530 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no reasoning. Its $41,500 is above even the roughly $36,767 tax under the standard deduction, so it skipped the $68,610 of itemized deductions and also overstated the bracket tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied post-TCJA-sunset law, with a $5,427 personal exemption and pre-TCJA brackets indexed from 2017. It also left Maryland income tax out of SALT. Current law keeps the TCJA brackets with no exemption and a $40,400 SALT cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired and applied a $5,300 personal exemption and the 10/15/25/28% brackets. OBBBA made the TCJA rates permanent with no exemption, and SALT is capped at $40,400."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and county income tax out of SALT, so it deducted $26,460 instead of the $40,400 cap. It also used wrong lower-bracket thresholds, giving $1,220 and $4,485 instead of $1,240 and $4,560."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Used TCJA-sunset law, with uncapped SALT of $36,346, a $5,434 personal exemption, and pre-TCJA brackets. Under OBBBA, 2026 has no exemption, SALT is capped at $40,400, and the 10/12/22/24% brackets apply."
+us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only real estate taxes in SALT and left out the Maryland state and local income tax, which pushes SALT to the $40,400 cap. Taxable income came out about $145,467 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly identified the $40,400 SALT cap but put only real estate taxes under it, leaving out the Maryland state and county income tax that brings SALT up to the cap. Itemized deductions came out $54,670 instead of $68,610."
+us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Entered $0 without any calculation. A single filer with $200,137 of AGI and $68,610 of itemized deductions has $131,526.91 of taxable income and owes $24,164.46."
+us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT, so SALT stopped at $26,460 instead of the $40,400 cap. That understated itemized deductions by $13,940."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the old $10,000 SALT cap instead of $40,400, and in the end used 2024 bracket thresholds instead of the 2026 indexed brackets."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Invented an AMT liability. Tentative minimum tax here is about 26% × ($171,927 AMTI − $90,100 exemption) ≈ $21,275, which is below the regular tax, so no AMT applies. It also mixed the standard deduction with itemized deductions and left Maryland income tax out of SALT."
us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household."
-us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the correct answer twice in its own reasoning — $184,500 wage base x 6.2% = $11,439 plus $198,505 x 1.45% = $2,878.32, total $14,317.32 — and then submitted $12,240.44, a figure its stated arithmetic never produces. The error is purely in the transfer of its own computed total to the value field, not in the payroll tax rules."
-us,scenario_078,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model applied the 2020 Social Security taxable wage base of $137,700 instead of the 2026 base of $184,500, understating the 6.2% component by $2,901.60. It then invented a Maryland employee payroll tax, first asserting Maryland has none and then adding an unexplained $3,765.20 'state contribution' at a made-up 3.75% effective rate; Maryland levies no mandatory employee payroll tax in 2026."
-us,scenario_078,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model's reasoning states the correct components — $184,500 x 6.2% = $11,439, $198,505 x 1.45% = $2,878.32, no Additional Medicare Tax below the $200,000 threshold, no Maryland employee payroll tax — and concludes 'Total = 14317.32.' It then submitted $12,550.53, a value inconsistent with every line of its own derivation."
-us,scenario_078,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model computed the correct $14,317.32, then second-guessed the wage base down to $183,600 for an alternative total of $14,261.52, and finally submitted $11,611.86 — a number matching neither of its two derivations. The 2026 Social Security wage base is $184,500, so its first computation was the correct one and it discarded it."
-us,scenario_078,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model used the 2025 Social Security taxable wage base of $176,100 rather than the 2026 base of $184,500, computing $10,918.20 of employee OASDI instead of $11,439. Every other element — $2,878.32 of Medicare, zero Additional Medicare Tax below the $200,000 threshold, no Maryland employee payroll tax — is correct, so the entire $520.80 shortfall is the stale wage base."
-us,scenario_078,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model applied the 2025 OASDI wage base of $176,100 instead of the 2026 base of $184,500, yielding $10,918.20 of Social Security tax rather than $11,439. It also rounded its $13,796.52 subtotal to $13,797, so its submitted value differs from its own stated arithmetic by $0.48 on top of the $520.80 wage-base shortfall."
-us,scenario_078,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used an approximate 2026 wage base of $184,000 instead of the actual $184,500, producing $11,408 of Social Security tax rather than $11,439. It then rounded the correct $14,286.32 subtotal to whole dollars, compounding the $31 wage-base error with a $0.32 rounding loss."
-us,scenario_078,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model added a Maryland FAMLI employee contribution of 0.45% x $183,000 = $823.50; Maryland's paid family and medical leave employee contributions are not collected in 2026, so no mandatory employee state payroll tax applies. It compounded this with a $183,000 wage base instead of the 2026 base of $184,500, understating Social Security tax by $93."
-us,scenario_078,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model added an $799.20 Maryland FAMLI employee contribution (0.45% of the wage cap) on top of federal FICA; Maryland collects no mandatory employee payroll contribution in 2026. It also used a projected wage cap of $177,600 rather than the actual 2026 base of $184,500, understating Social Security tax by $427.80."
-us,scenario_078,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model applied the 2024 Social Security wage base of $168,600 rather than the 2026 base of $184,500, computing $10,453 of OASDI instead of $11,439. Its own stated components sum to $13,331, yet it submitted $14,193, so the value also fails to match its own arithmetic."
-us,scenario_078,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model added 'estimated state-level payroll contributions (MD FAMLI)' to federal FICA; Maryland levies no mandatory employee payroll contribution in 2026, and its $15,114 exceeds the correct $14,317.32 by $796.68, almost exactly 0.45% of the wage base. The federal portion of its answer tracks the correct rates and cap, so the entire excess is the fabricated Maryland contribution."
-us,scenario_078,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used an invented 2026 Social Security wage limit of $181,800 instead of the actual $184,500, producing $11,271.60 of OASDI rather than $11,439. Its Medicare computation of $2,878.32 is correct, so the $167.40 wage-base shortfall (plus $0.08 of rounding to $14,150) is the entire error."
-us,scenario_078,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model named the wage base in its explanation but never applied it: its $15,234.39 exceeds even the uncapped 7.65% of $198,505 ($15,185.63), which is only reachable by taxing all wages at 6.2% with no cap. Capping OASDI at the 2026 base of $184,500 yields $11,439, and adding $2,878.32 of Medicare gives $14,317.32."
-us,scenario_078,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model asserted a 2026 Social Security cap of $181,800 rather than the actual $184,500, computing $11,271.60 of OASDI instead of $11,439. Its $2,878.32 Medicare figure is exact, so the full $167.40 gap is the incorrect wage base."
-us,scenario_078,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model applied the correct 6.2% and 1.45% rates but used a wage base of about $181,861 — its $14,153.62 less the exact $2,878.32 Medicare leaves $11,275.30 of OASDI — instead of the 2026 base of $184,500. The correct capped OASDI is $11,439, so the $163.70 shortfall is entirely the wage base."
-us,scenario_078,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model's $14,194 less the correct $2,878.32 Medicare implies an OASDI figure of $11,315.68, i.e. a wage base near $182,511 rather than the 2026 base of $184,500. Applying the actual base yields $11,439 of Social Security tax and a $14,317.32 total."
-us,scenario_078,payroll_tax,glm-5.2,llm_error,other,False,"The model used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, deriving $13,796.52, and then submitted $11,870.18, which matches neither that derivation nor the correct $14,317.32. Both errors compound: a stale wage base costing $520.80 and a submitted value $1,926.34 below its own stated total."
-us,scenario_078,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct method — 6.2% up to the wage base plus 1.45% on all wages, no Additional Medicare Tax, no Maryland employee tax — but its $15,463 exceeds even the uncapped 7.65% of $198,505 ($15,185.63), so it applied no wage base at all and produced a number above the maximum any FICA rate structure yields here. Capping OASDI at $184,500 gives $11,439 plus $2,878.32 of Medicare, for $14,317.32."
-us,scenario_078,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model described applying the combined employer-plus-employee rates of 12.4% for Social Security and 2.9% for Medicare, which are the self-employment rates, rather than the employee-side 6.2% and 1.45% the question requires. Its submitted $13,508 matches neither those doubled rates nor the correct employee computation of $11,439 + $2,878.32 = $14,317.32."
-us,scenario_078,payroll_tax,gpt-5.5,llm_error,other,False,"The model assumed a 2026 wage base of $183,900 rather than $184,500, computing $11,401.80 + $2,878.32 = $14,280.12, then submitted $11,816.31 while trailing off into a question about a Maryland paid family leave rule. Maryland collects no employee paid-leave contribution in 2026, and the submitted value is $2,463.81 below the model's own arithmetic."
-us,scenario_078,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The correct derivation is $184,500 x 6.2% = $11,439 of Social Security tax plus $198,505 x 1.45% = $2,878.32 of Medicare tax, totaling $14,317.32. The model's $11,485 sits within $46 of the Social Security component alone, consistent with reporting only capped OASDI and dropping the uncapped 1.45% Medicare tax entirely."
-us,scenario_078,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model used an approximate 2026 wage base of $183,700 instead of the actual $184,500, yielding $11,389 of Social Security tax rather than $11,439. It also rounded both components to whole dollars, so its $14,268 falls $49.32 short of the correct $14,317.32."
-us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so the required key was absent from its submitted outputs. No substantive payroll tax reasoning exists to evaluate."
-us,scenario_078,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with the bare explanation 'value = 0' for a wage earner with $198,505 of covered wages, which carry mandatory 6.2% OASDI up to the $184,500 base and uncapped 1.45% Medicare. No exemption in the household facts removes FICA from these wages; the correct total is $11,439 + $2,878.32 = $14,317.32."
-us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model projected a 2026 wage base of about $174,900 by inflating the 2024 base of $168,600, skipping the actual 2025 base of $176,100 and the 2026 base of $184,500. That understated OASDI at $10,843.80 instead of $11,439, and the resulting $13,722.12 falls short by exactly that $595.20."
-us,scenario_078,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated Social Security tax of $10,453.20, which is 6.2% of the 2024 wage base of $168,600 rather than the 2026 base of $184,500, and its stated components sum to $13,331.52. It nonetheless submitted $15,622.81, a value $2,291.29 above its own arithmetic and above the correct $14,317.32."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took the ~$2,700 Maryland standard deduction and an $800 exemption instead of itemizing $54,670 of mortgage interest and property tax, which put about $195,000 into the 5.5% bracket. It then subtracted those same allowances again from its $9,735 bracket result to reach $9,327. The correct base is $198,505 minus $54,659.73 in itemized deductions with no exemption above $150,000 AGI."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the refund and itemized $54,670. It then applied a $600 personal exemption, but Maryland's single-filer exemption is $0 above $150,000 AGI. It also skipped the $10.27 phase-out (7.5% of federal AGI over $200,000), so its taxable income of $143,235 is $610.27 too low, a $32.04 shortfall at 5.25%."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It took a $3,200 Maryland standard deduction, leaving about $195,305 of taxable income, and never itemized the $54,670 of mortgage interest and property tax. Its $7,558 also does not match its own base: the Maryland brackets on $195,305 give about $9,776."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached $6,936.59 on taxable income of about $143,835 by itemizing correctly, then dropped that result for an unsupported $9,755. That figure matches taxing about $195,000, i.e. the standard-deduction path with no itemizing. It also skipped the $10.27 itemized phase-out."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It capped property tax at a $10,000 SALT limit, using about $30,000 of itemized deductions and $168,505 of taxable income. The full $26,460 is deductible because it sits under the 2026 $40,400 cap. It then submitted $9,876, which contradicts its own $8,277.78 bracket calculation."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $200,137 without subtracting the $1,632 state refund and itemized only about $38,210, capping property tax at $10,000. The full $26,460 property tax is deductible. It also used a wrong 5.0% rate for the $150,000-$175,000 range, which produced its $161,900 base and $7,800 tax."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from unadjusted federal AGI of $200,137, giving $194,537. It never itemized the $54,670 of mortgage interest and property tax, never subtracted the state refund, and gave an exemption that is $0 above $150,000 AGI."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached about $143,835 of Maryland taxable income after the $54,670 itemized deduction, but it never ran the brackets on that base. It guessed $9,200, an average rate of about 6.4%, even though the top marginal rate at this income is 5.25%. The brackets give about $6,936."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used Maryland AGI of $200,137 without subtracting the $1,632 state tax refund, which Maryland exempts, and skipped the $10.27 phase-out. That gave taxable income of $145,467 and tax of $7,022 instead of $143,845.27 and $6,936.88."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped property tax at $10,000 (itemized $38,210 instead of $54,670), skipped the refund subtraction, and took a $3,200 exemption that is $0 above $150,000 AGI. That inflated taxable income to $158,727. Its bracket math on that base is correct."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from $200,137, giving $194,537. It never itemized the $54,670 of mortgage interest and property tax and never subtracted the state refund."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It got Maryland AGI ($198,505) and the $0 exemption right but cut itemized deductions to $44,642.60, a reduction of $10,027. Maryland's phase-out is 7.5% of federal AGI over $200,000, which removes only $10.27 here. The over-cut raised taxable income to $153,862 and the tax to $7,472."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It skipped the refund subtraction (used $200,137) and reduced itemized deductions by a federal-style 3% of AGI over $200,000 ($4.11) instead of Maryland's 7.5% ($10.27). It also took a $1,600 exemption that is $0 above $150,000 AGI, so taxable income came to $143,871.11 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $9,070 matches taxing about $183,000 at Maryland's 5.5% top bracket, i.e. income reduced only by a standard-type deduction. It never subtracted the $54,670 of mortgage interest and property tax that bring taxable income down to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It says it included county/local income tax, which the requested output excludes. Its $11,406 is the $6,936.88 state tax plus roughly $4,470 of county tax at about 3.1% of taxable income."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached taxable income of $143,835 by itemizing with no exemption, but misapplied the brackets: 2%/3%/4%/4.75%/5%/5.25% on $143,835 gives $6,936.34, not $6,886. It also skipped the $10.27 phase-out, which raises taxable income to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It says it used standard deductions and exemptions, and its $9,876 matches taxing about $197,500 in the 5.5% bracket. It never itemized the $54,670 of mortgage interest and property tax, which cut taxable income to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly computed the state tax as $6,936.34 on $143,835, then added a 3.2% county tax of $4,602.72, which this output excludes. It also skipped the $10.27 itemized phase-out."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It took the right path (itemized mortgage interest and property tax, zero exemption) but its $6,889 is $47 below the bracket tax on that base ($6,936.34 on $143,835), a bracket arithmetic error. It also skipped the $10.27 phase-out."
-us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It skipped the refund subtraction, so taxable income was $145,467. It then misapplied the brackets: 2%/3%/4%/4.75%/5%/5.25% on that base gives $7,022.02, not $7,300.69. Its claim that the SALT cap expires in 2026 is also wrong; the 2026 cap is $40,400."
-us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used $38,210 of itemized deductions, capping property tax at $10,000, instead of the full $26,460. It then set up a circular add-back of state income tax that no one paid and taxed the result at a flat 4.75%, skipping the 5% and 5.25% brackets."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It cites a standard allowance, and its $9,720 matches taxing about $194,500 ($200,137 minus a $2,400 standard deduction and a $3,200 exemption). It never itemized the $54,670 of mortgage interest and property tax and never subtracted the state refund."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $10,120 is higher than the Maryland bracket tax on the full $200,137 with no deductions at all (about $10,018). So it applied neither the $54,659.73 itemized deduction nor the correct 2%-5.5% schedule."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $6,511.18 corresponds to taxable income of about $135,700, roughly $8,100 below the correct $143,845.27. It took deductions beyond the $54,659.73 of mortgage interest and property tax (after the phase-out), including a personal exemption that is $0 for a single filer above $150,000 AGI."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,025 matches taxing about $145,500, i.e. federal AGI of $200,137 minus $54,670 of itemized deductions. It skipped Maryland's subtraction of the $1,632 state tax refund and the $10.27 phase-out."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used taxable income of $145,467, starting from $200,137 without subtracting the $1,632 state tax refund that Maryland exempts. It also skipped the $10.27 phase-out, so its $7,022.02 is $85 too high."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. Its $9,200 matches taxing about $185,000 in the 5.5% bracket, roughly AGI minus a standard deduction. It never subtracted the $54,670 of mortgage interest and property tax, which bring taxable income to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction from $200,137, giving $197,737 taxed up to 5.5%. It never itemized the $54,670 of mortgage interest and property tax and never subtracted the $1,632 state tax refund."
-us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It cut itemized deductions by 3% of AGI over $100,000 ($3,004) instead of Maryland's 7.5% of federal AGI over $200,000 ($10.27). It also kept a $576 partial exemption that is $0 above $150,000 and skipped the refund subtraction, so taxable income came to $147,895 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so no answer was scored."
-us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 with no reasoning, as if no Maryland income tax were owed. With $198,505 of wages and $143,845.27 of taxable income after itemizing, the Maryland brackets give $6,936.88."
-us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It taxed everything above $3,000 at a flat 4.75%, skipping the 5% bracket above $100,000 and the 5.25% bracket above $125,000. It also left the $1,632 state refund in Maryland AGI, which gave taxable income of $145,467."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took only a ~$2,550 standard deduction and never itemized the $54,670 of mortgage interest and property tax. Its $10,933 is also higher than its own bracket computation on $195,955 (about $9,788) and even its $9,927.78 on full wages."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income of $184,037, which is after the federal standard deduction, instead of Maryland AGI of $198,505 less $54,659.73 of Maryland itemized deductions. It used a made-up schedule (5% on $95,000, then 5.25% on the rest) and subtracted a nonrefundable credit when none applies."
+us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning gets the correct total of $14,317.32 ($11,439 SS on the $184,500 base plus $2,878.32 Medicare), but it submitted $12,240.44. Nothing in its derivation supports that figure, so the submitted value contradicts its own correct computation."
+us,scenario_078,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It capped Social Security at the 2020 wage base of $137,700 instead of the 2026 base of $184,500, which understates SS tax by $2,901.60. It also added a made-up Maryland employee payroll tax of about 3.75% ($3,765.20). Maryland's income tax is not a payroll tax, and Maryland has no mandatory employee payroll tax in 2026."
+us,scenario_078,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It said it did not compute employee Social Security or Medicare tax and entered 0. That skips the required 6.2% SS tax on $184,500 ($11,439) and the 1.45% Medicare tax on $198,505 ($2,878.32)."
+us,scenario_078,payroll_tax,claude-opus-4.7,llm_error,other,False,"Its reasoning gets the correct $14,317.32 ($11,439 SS on the $184,500 base plus $2,878.32 Medicare), but it submitted $12,550.53, which no step of its derivation produces. The submitted value contradicts its own correct total."
+us,scenario_078,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It first got the correct $14,317.32 using the $184,500 base, then switched to a wrong $183,600 base ($14,261.52). It then submitted $11,611.86, which matches neither calculation. The final value drops the correct 2026 wage base and is not tied to any stated arithmetic."
+us,scenario_078,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That gives $10,918.20 of SS tax instead of $11,439, which understates the total by $520.80."
+us,scenario_078,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the 2025 OASDI wage base of $176,100 instead of the 2026 base of $184,500. That puts SS tax at $10,918.20 instead of $11,439, and it then rounded the $13,796.52 total to $13,797."
+us,scenario_078,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $184,000 wage base instead of the actual 2026 base of $184,500, which understates SS tax by $31 ($11,408 vs $11,439). It also rounded the total down to $14,286."
+us,scenario_078,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a $183,000 wage base instead of $184,500. It also added a 0.45% Maryland FAMLI employee contribution ($823.50), but no FAMLI contributions are collected in 2026, so no state employee payroll tax applies."
+us,scenario_078,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It projected the 2026 SS wage base at $177,600 instead of $184,500. It also added a $799.20 Maryland FAMLI employee contribution, which does not apply because Maryland collects no FAMLI payroll contribution in 2026."
+us,scenario_078,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. Its own components ($10,453 + $2,878 = $13,331) also do not add up to the $14,193 it submitted."
+us,scenario_078,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added an estimated Maryland FAMLI employee contribution to SS and Medicare, which overstates the total by about $797. Maryland collects no employee FAMLI payroll contribution in 2026, so the answer is just $11,439 + $2,878.32."
+us,scenario_078,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used an estimated 2026 wage base of $181,800 instead of the actual $184,500. That gives $11,271.60 of SS tax instead of $11,439, and it then rounded the total to $14,150."
+us,scenario_078,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It never applied the $184,500 Social Security wage cap. Its $15,234.39 is even higher than uncapped 6.2% + 1.45% on $198,505 ($15,185.63), so the SS part was not capped and the arithmetic is wrong. The correct calculation is 6.2% × $184,500 + 1.45% × $198,505."
+us,scenario_078,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"It capped Social Security at a projected $181,800 instead of the actual 2026 wage base of $184,500. That gives $11,271.60 of SS tax instead of $11,439, which understates the total by $167.40."
+us,scenario_078,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $14,153.62 means an SS component of $11,275.30, which matches a wage base of about $181,860 instead of the 2026 base of $184,500. That understates the 6.2% Social Security tax."
+us,scenario_078,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $14,194 means an SS component of about $11,316, which matches a wage base of roughly $182,500 instead of the 2026 base of $184,500. That understates the 6.2% Social Security tax by about $123."
+us,scenario_078,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"It used the 2025 taxable maximum of $176,100 instead of the 2026 base of $184,500, which gives a $13,796.52 total. It then submitted $11,870.18, which does not match even its own wrong total."
+us,scenario_078,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It described the right structure but did not apply the $184,500 wage cap. Its $15,463 implies an SS component of about $12,585, which is more than 6.2% of the full $198,505 of wages. The correct total is $11,439 SS plus $2,878.32 Medicare."
+us,scenario_078,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It used the combined employer-plus-employee rates (12.4% SS and 2.9% Medicare) instead of the employee-only rates of 6.2% and 1.45%. Its $13,508 also does not come from any shown calculation. The employee-only answer is 6.2% × $184,500 + 1.45% × $198,505."
+us,scenario_078,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It used a $183,900 wage base instead of $184,500, giving $14,280.12. It then subtracted about $2,464 for a Maryland paid family/medical leave rule that it admitted is not in effect. A program that is not in effect cannot reduce federal employee FICA, so the right answer is simply $11,439 + $2,878.32."
+us,scenario_078,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"Its $11,485 is about $2,832 short, which is nearly the full $2,878.32 Medicare tax. This matches reporting only the capped 6.2% Social Security tax (about $11,439) and leaving out the 1.45% Medicare tax on all $198,505 of wages."
+us,scenario_078,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"It used a 2026 wage base of about $183,700 instead of $184,500, which gives $11,389 of SS tax instead of $11,439. It also rounded Medicare to $2,878, for a total of $14,268."
+us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation, so there is no answer to score. The correct value is $11,439 SS (6.2% × $184,500) plus $2,878.32 Medicare (1.45% × $198,505)."
+us,scenario_078,payroll_tax,minimax-m3,llm_error,missing_output,False,"It entered 0 even though the household has $198,505 in wages, which leaves out all employee FICA. The correct answer is 6.2% Social Security on $184,500 ($11,439) plus 1.45% Medicare on all wages ($2,878.32)."
+us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It projected the 2026 wage base as $174,900 by inflating the 2024 figure, instead of using the actual 2026 base of $184,500. That understates SS tax by $595.20 ($10,843.80 vs $11,439)."
+us,scenario_078,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It capped Social Security at the 2024 base of $168,600 ($10,453.20) instead of the 2026 base of $184,500. It then submitted $15,622.81, which does not match its own components ($10,453.20 + $2,878.32 = $13,331.52)."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a ~$2,700 Maryland standard deduction and an $800 exemption, giving about $195,005 of taxable income. It should have itemized $54,670 in mortgage interest and real estate taxes, less the $10.27 phaseout. A single filer with Maryland AGI above $150,000 gets no exemption at all, so the correct taxable income is $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its structure was right, but it allowed a $600 personal exemption, which is $0 for a single filer with Maryland AGI above $150,000. It also skipped the itemized deduction reduction of 7.5% of federal AGI over $200,000 ($10.27). Together these understated taxable income by $610.27, or $32.04 of tax at the 5.25% rate."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied a $3,200 Maryland standard deduction to reach about $195,305 of taxable income. It should have itemized: Maryland allows the $28,210 mortgage interest and the full $26,460 in property taxes, less a $10.27 phaseout. Its $7,558 also does not follow from its own taxable income figure under Maryland's brackets."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed Maryland tax and entered $0. The correct path is Maryland AGI of $198,505, minus $54,659.73 of itemized deductions and a $0 exemption, which gives $143,845.27 taxed at Maryland rates for $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning correctly reached $143,835 of taxable income and $6,936.59 of tax, missing only the $10.27 phaseout. It then dropped that result and submitted an unsupported $9,755, which is consistent with taxing almost all of Maryland AGI without the itemized deductions."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed Maryland itemizing follows a $10,000 SALT cap and used only ~$30,000 of itemized deductions. Maryland itemized deductions include the full $26,460 in real estate taxes plus $28,210 in mortgage interest; only state income taxes are removed. It then submitted $9,876 even though its own bracket calculation came to $8,277.78."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left the $1,632 refund in Maryland AGI ($200,137 instead of $198,505). It also capped property taxes at $10,000, which made itemized deductions ~$38,210 instead of $54,659.73. On top of that it used a wrong rate schedule with a 5.0% top bracket for $150k–$175k."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from $200,137. It should have subtracted the $1,632 refund to get Maryland AGI and itemized $54,670 in mortgage interest and property taxes, less the $10.27 phaseout. The exemption is $0 for a single filer above $150,000."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached about $143,835 of taxable income using $54,670 of itemized deductions. It then guessed $9,200 instead of applying the brackets: $90 on the first $3,000, 4.75% to $100k, 5% to $125k and 5.25% on the rest, which gives about $6,937."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $200,137 as Maryland AGI without subtracting the $1,632 state tax refund, which Maryland excludes. It also skipped the $10.27 itemized phaseout. That left taxable income at $145,467 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped real estate taxes at $10,000, giving $38,210 of itemized deductions instead of the full $54,670 less $10.27. It claimed a $3,200 exemption, which is $0 above $150,000 for a single filer. It also kept the $1,632 refund in Maryland AGI."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $2,400 standard deduction and a $3,200 exemption on $200,137. It should have itemized $54,659.73, subtracted the $1,632 refund from Maryland AGI, and allowed no exemption, since Maryland AGI is above $150,000."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Its Maryland AGI ($198,505) and $0 exemption were correct. But it cut the $54,670 in itemized deductions by about $10,027 to $44,642.60. Maryland's phaseout is only 7.5% of federal AGI over $200,000, which here is $10.27, so taxable income should be $143,845.27, not $153,862.40."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left the $1,632 refund in Maryland AGI and used a 3% phaseout instead of 7.5% of federal AGI over $200,000. It also allowed a $1,600 exemption, which is $0 above $150,000. These errors partly cancelled, giving taxable income of $143,871.11 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $9,070 corresponds to about $183,000 of Maryland taxable income, which means only a standard-deduction-sized amount was subtracted. It ignored the $54,670 itemized deduction for mortgage interest and real estate taxes, which brings taxable income down to $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added county/local income tax to the state amount, but this output covers state tax only and excludes local tax. The state tax alone, on $143,845.27 after itemizing, is $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached roughly the right taxable income ($143,835, missing only the $10.27 phaseout) but got the bracket arithmetic wrong. $90 + $4,607.50 + $1,250 + 5.25% × $18,835 = $6,936.34, not $6,886."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It explicitly used standard deductions. $9,876 corresponds to about $197,500 of taxable income, so it never itemized the $54,670 in mortgage interest and property taxes that Maryland allows."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It computed the state tax correctly at $6,936.34, then added a 3.2% county income tax ($4,602.72). That local tax belongs in a separate output and is excluded from state income tax before refundable credits."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly itemized with a $0 exemption, which gives about $143,845 of taxable income. Its $6,889 is about $48 short, which means it misapplied the brackets. The correct bracket tax is $90 + $4,607.50 + $1,250 + 5.25% × $18,845.27 = $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It kept the $1,632 refund in Maryland AGI, giving $145,467 of taxable income. It then mis-applied the brackets: the correct tax on $145,467 is $7,022, not $7,300.69. The right base is $143,845.27, which gives $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It capped property taxes at $10,000 (federal itemized of $38,210) and ran a circular state-tax addback. Maryland allows the full $26,460 since the 2026 cap is $40,400 and no state income tax was paid. It also used a flat 4.75% rate, ignoring the 5% and 5.25% brackets above $100,000 and $125,000."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It applied a standard allowance. $9,720 corresponds to about $195,000 of taxable income, so it never subtracted the $54,659.73 in Maryland itemized deductions for mortgage interest and real estate taxes."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"$10,120 is more than Maryland's own brackets produce on the full $198,505 of AGI. So it neither applied the $54,659.73 itemized deduction nor used the correct rate schedule, which gives $6,936.88 on $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It claimed a personal exemption, which is $0 for a single filer above $150,000. $6,511.18 corresponds to about $135,700 of taxable income, roughly $8,100 below the correct $143,845.27, so it took deductions beyond the $54,659.73 Maryland allows."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$7,025 corresponds to about $145,500 of taxable income. That means it itemized from federal AGI of $200,137 without subtracting the $1,632 refund that Maryland excludes, and without the $10.27 phaseout."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used $145,467 of taxable income, starting from federal AGI of $200,137. It should have subtracted the $1,632 state tax refund to get Maryland AGI of $198,505 and applied the 7.5% phaseout ($10.27), for taxable income of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. $9,200 corresponds to about $185,000 of taxable income, so only a small standard-type deduction was subtracted. It skipped the $54,659.73 itemized deduction, which brings taxable income to $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took the $2,400 standard deduction from $200,137. It should have subtracted the $1,632 refund to get Maryland AGI and itemized $54,670 in mortgage interest and real estate taxes, less the $10.27 phaseout, for taxable income of $143,845.27 instead of $197,737."
+us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It cut itemized deductions by 3% of AGI over $100,000 ($3,004), but Maryland's limit is 7.5% of federal AGI over $200,000 ($10.27). It also kept a $576 exemption that is fully phased out above $150,000, and left the $1,632 refund in Maryland AGI."
+us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"There was no parseable value for state_income_tax_before_refundable_credits, so no Maryland tax figure was submitted."
+us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It entered $0 with no derivation, so it never applied Maryland tax to $143,845.27 of taxable income (Maryland AGI of $198,505 minus $54,659.73 of itemized deductions). That income is taxed at $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a flat 4.75% on everything above $3,000, leaving out the 5% bracket ($100k–$125k) and the 5.25% bracket ($125k–$150k). It also kept the $1,632 refund in Maryland AGI, giving $145,467 of taxable income instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a ~$2,550 standard deduction instead of itemizing the $54,670 in mortgage interest and real estate taxes. Its $10,933 is also higher than its own $9,927.78 bracket calculation on the larger base, which is internally inconsistent."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income less a Maryland standard deduction instead of Maryland AGI ($198,505) less Maryland itemized deductions ($54,659.73). It used a wrong bracket schedule (5% on $95,000 and 5.25% on everything above $100,000). It also subtracted an invented nonrefundable credit, when none applies."
us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_079,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model tested only Arizona's income-based adult (MAGI) Medicaid rule and declared the head failed it, never applying the SSI-recipient category: the head's blindness, disability, and $7,560 of SSDI produce $5,286 of SSI, and Arizona is a 1634 state where SSI approval automatically confers AHCCCS/Medicaid with no separate income or resource test. Its own stated route also cannot yield 'No' — countable income at 0.35 x FPL sits far below Arizona's 138% FPL expansion limit — so the model both skipped the operative categorical pathway and misapplied the fallback one it invoked."
us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed pathway or coverage input and returned 0 for lack of one, but SSI receipt is computed from the facts given (age 58, blind, disabled, $7,560 SSDI, $1,045 in resources, no other income), and that SSI receipt is itself the categorical Medicaid pathway in Arizona, which auto-enrolls SSI recipients. It compounded this by reading Indian Health Service coverage at interview as the head's coverage source; IHS is not minimum essential coverage and never displaces or disqualifies Medicaid eligibility."
@@ -6343,141 +6509,145 @@ us,scenario_079,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disabilit
us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read the 'constant throughout the year' instruction as satisfying the 24-month SSDI requirement, but constant receipt within 2026 gives only 12 months. Prior months are unlisted and default to 0, so Head does not qualify."
us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted only the head's $630/month SSDI and left out the $881/month of SSI that the blind/disabled couple receives, which is countable unearned income for SNAP. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Its final $380/month doesn't even match its own $417.50 arithmetic; with $1,511 gross and $1,157 net income, the benefit is $546 − $348 = $198/month."
-us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI, the $209 standard deduction, and the $546 maximum allotment. But it deducted only the $26 actual medical excess ($61 − $35) instead of Arizona's $145 standard medical deduction for elderly/disabled households with over $35 of medical costs. That left net income at $1,276 instead of $1,157 and cut the benefit to $163 instead of $198/month."
-us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It ruled the household ineligible on resources by counting home and vehicle equity. The home is always an excluded resource and Arizona SNAP does not count vehicles, so countable assets are only the $1,045 bank balance, which passes. It never counted the $881/month SSI or computed the benefit, which is $198/month on $1,157 of net income."
-us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $630/month SSDI as all of the household's income and left out the couple's $881/month SSI. It also used a $219 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction. Its final $351.50/month is an unexplained adjustment away from its own $420.50 arithmetic; correct inputs give $1,157 net income and $198/month."
-us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted only the $630/month SSDI and left out the $881/month SSI, then guessed net income 'near $300' instead of applying the $209 standard deduction and Arizona's $145 standard medical deduction to $1,511 of gross income. Its $303/month benefit is about $105 above the correct $198."
-us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the couple's $881/month SSI, so gross income was $630 instead of $1,511, and it did not apply Arizona's $145 standard medical deduction. Its $283/month answer doesn't follow from its own $536 − 30% × ~$400 arithmetic. Correct net income is $1,157, giving $198/month."
-us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI and used the $209 standard deduction and $546 maximum. But it deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. That overstated net income by $119 ($1,276 vs $1,157) and understated the benefit at $163 instead of $198/month."
-us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It acknowledged that the members receive SSI/SSDI, but then counted only the $630/month SSDI and left out the $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, plus outdated $204 standard-deduction and $535 maximum-allotment figures. Correct inputs give $1,157 net income and $198/month, not $415."
-us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left out the $881/month SSI and inferred large shelter costs from the $455,000 mortgage balance, even though no mortgage payment, interest, or housing cost was listed. It then used $374 as the two-person maximum instead of $546. With no shelter deduction and Arizona's $145 standard medical deduction, net income is $1,157 and the benefit is $198/month."
-us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It treated the $630/month SSDI as the only income and left out the couple's $881/month SSI, which is countable unearned income for SNAP. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. With $1,511 gross and $1,157 net income, the benefit is $198/month, not $427.50."
-us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $228 standard deduction (the correct figure is $209), a $549 maximum (correct is $546), and SSI of $876 (correct is $881). It also skipped rounding the $375.60 contribution up to a whole dollar. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month."
-us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"It applied the federal $4,650 vehicle fair-market-value exclusion and counted $5,050 of vehicle equity. Arizona SNAP does not count vehicles, so countable resources are only the $1,045 bank balance and the household passes the asset test. With SSI counted and Arizona's $145 standard medical deduction, the benefit is $198/month."
-us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It counted SSI (at $872 instead of $881) but deducted only the $26 actual medical excess ($732/yr − $420/yr) instead of Arizona's $145 standard medical deduction. It also used a $202 standard deduction and a $548 maximum instead of $209 and $546. Correct net income is $1,157, giving $198/month, not $165.80."
-us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed net income from the $630 SSDI alone and left out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct gross income is $1,511 and net income $1,157, so the benefit is $198/month, not $427."
-us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"It counted SSI, though at $805 instead of $881/month, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $198 standard deduction and a $535 maximum instead of $209 and $546. Correct net income is $1,157 and the benefit is $198/month."
-us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It concluded SNAP is zero because no rent or utilities were reported. Shelter costs only drive the optional excess shelter deduction; a household with no housing costs still gets the maximum allotment minus 30% of net income. Net income of $1,157 gives $546 − $348 = $198/month."
-us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It claimed that 30% of net income exceeds the maximum allotment and paid only a $23/month minimum benefit. After the $209 standard deduction and Arizona's $145 standard medical deduction, net income is $1,157, and 30% of that ($348) is well below the $546 maximum. The benefit is therefore $198/month, not the minimum."
-us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"It included SSI but deducted only the $312/year actual medical excess instead of Arizona's $1,740/year ($145/month) standard medical deduction. It also used a $2,400 standard deduction and a $6,720 maximum instead of $2,508 and $6,552. Correct annual net income is $13,884, giving $2,376."
-us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"It asserted asset and income disqualification without computing either. Countable assets are only the $1,045 bank balance because the home and vehicles are excluded, and net income of $1,157 is below the $1,762.50 poverty line. The household qualifies for $198/month."
-us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"It counted SSDI plus SSI ($18,096/yr) but, per its own description, used the actual medical excess ($26/month) instead of Arizona's $145 standard medical deduction for elderly/disabled households. That overstated net income by about $119/month and gave $178 instead of $198/month."
-us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $470/month answer implies net income of about $250, which is only reachable by counting the $630 SSDI alone and leaving out the couple's $881/month SSI. Counting $1,511 of gross income, the $209 standard deduction, and the $145 Arizona medical deduction gives $1,157 net income and $198/month."
-us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted SSI, though at the 2025-based $840/month instead of $881. But it deducted only $12/year of medical costs by counting just the $432 of other medical expenses (dropping the $300 of over-the-counter costs), instead of Arizona's $145/month standard medical deduction. It also used $204 and $536 instead of the $209 standard deduction and $546 maximum."
-us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"It returned no SNAP value or explanation, so there was no substantive answer to score. The correct derivation is $1,511 gross income minus $209 standard and $145 Arizona medical deductions, giving $1,157 net and $198/month ($2,376/year)."
-us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed net income from the $7,560 of SSDI alone and left out the couple's $10,572/year of SSI. It also used the $312/year actual medical excess instead of Arizona's $1,740/year standard medical deduction, and a $6,612 maximum instead of $6,552. Correct annual net income is $13,884, so SNAP is $2,376, not $5,172."
-us,scenario_079,snap,gpt-5.4-mini,llm_error,other,False,"It set SNAP to zero because no dependents or shelter costs were listed. SNAP doesn't require either: a two-person disabled household gets the $546 maximum minus 30% of net income. With $1,157 of net income, the benefit is $198/month."
-us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned zero on the claim that the listed facts don't support a SNAP computation. The facts fully determine the benefit: $630 SSDI plus $881 SSI, minus the $209 standard deduction and the $145 Arizona standard medical deduction, gives $1,157 net income and $198/month."
-us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI and used the $546 maximum, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction instead of $209. That gave net income of $1,281 instead of $1,157 and $161 instead of $198/month."
-us,scenario_079,snap,gpt-5.6-luna,llm_error,other,False,"It paid the full $546 maximum allotment every month and never subtracted the expected contribution of 30% of net income. With $1,511 of gross income and $1,157 of net income, that contribution is $348, leaving $198/month."
-us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163/month answer matches $1,511 gross income minus the $209 standard deduction and only the $26 actual medical excess. It missed Arizona's $145 standard medical deduction, which lowers net income to $1,157 and raises the benefit to $198/month."
-us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"It counted SSDI plus SSI but applied only the actual 'allowable' medical excess (about $26/month) instead of Arizona's $145 standard medical deduction for elderly/disabled households. That produced $165 instead of the $198/month that $1,157 of net income yields."
-us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"It correctly used $1,511 of income, the $209 standard deduction, and the $546 maximum, but subtracted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also skipped rounding the contribution up to a whole dollar. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month."
-us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $395 net income comes from the $630 SSDI alone, leaving out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct net income is $1,157, so the benefit is $198/month, not $427."
-us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI with the $209 standard deduction and $546 maximum, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. That gave net income of $1,276 instead of $1,157 and $163 instead of $198/month."
-us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"It used the right $1,511 of income, $209 standard deduction, and $546 maximum, but subtracted the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also left the 30% contribution unrounded. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month ($2,376/year)."
-us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"It asserted that the SSDI income exceeds SNAP limits for a household of two. Even with SSI included, gross income of $1,511 is 86% of the $1,762.50 poverty guideline, and net income of $1,157 easily passes the net income test. The household gets $198/month."
-us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It counted only $7,560 of SSDI and left out the couple's roughly $10,572/year of SSI. It also used the $312/year actual medical excess instead of Arizona's $145/month standard medical deduction, with a $2,520 standard deduction and a $6,600 maximum instead of $2,508 and $6,552. The correct annual benefit is $2,376, not $5,182."
-us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,"It included SSI ($1,506 total, close to the correct $1,511) and the $209 standard deduction, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $549 maximum instead of $546. Correct net income is $1,157, giving $198/month rather than $168."
-us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI with the $209 standard deduction and $546 maximum, but applied only $26 of actual excess medical costs instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also left the contribution unrounded. Correct net income is $1,157 and the benefit is $198/month."
-us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"It counted SSI at the 2025 rate ($840 instead of $881/month) and deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546. Correct net income is $1,157, giving $198/month rather than $164."
-us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"It treated $630/month of SSDI as the household's only income and left out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and a $548 maximum instead of $546. With $1,511 gross and $1,157 net income, the benefit is $198/month, not $429."
-us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value or explanation, so there was no substantive answer to score. The correct derivation is $1,511 gross income minus $209 standard and $145 Arizona medical deductions, giving $1,157 net and $198/month ($2,376/year)."
-us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed net income as $630 SSDI − $209 − $26 = $395, leaving out the couple's $881/month SSI and using the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct net income is $1,157, and $546 − $348 = $198/month."
-us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,"It returned zero on the grounds that benefit receipt shouldn't be inferred, even though the prompt tells it to assume program take-up and it had itself found income below the SNAP thresholds. Computing the benefit gives $1,511 gross, $1,157 net, and $198/month."
-us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted only the $630/month SSDI and left out the couple's $881/month SSI. It also used a ~$205 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction. Correct net income is $1,157, so the benefit is $198/month, not ~$427."
-us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left out the couple's SSI and used $7,560 of SSDI as total income. It applied the $35 medical threshold per year instead of per month and never used Arizona's $145/month standard medical deduction. It also used a $598 maximum and a $198 standard deduction instead of $546 and $209. Correct annual SNAP is 12 × ($546 − $348) = $2,376."
-us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated SSI receipt as ruling out separate SNAP, applying the old California SSI cash-out rule. Arizona SSI recipients are SNAP-eligible, and SSI receipt actually confers categorical eligibility. With SSI counted as income, the household receives $198/month."
+us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630 of monthly SSDI and never computed the couple's $881 of monthly SSI ($1,491 couple FBR minus $610 of countable SSDI), so gross income was $1,511 short by $881. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, then gave a self-contradictory $4,560 that its own arithmetic does not support."
+us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"Got the $1,511 income, the $209 standard deduction, and the $546 maximum right, but used the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households with costs above $35. That left net income at $1,276 instead of $1,157 and the benefit at about $163 a month instead of $198."
+us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"Denied SNAP on assets by counting home equity and vehicle equity, but the home is an excluded resource and PolicyEngine's asset test counts only the $1,045.29 bank balance, which is well under the limit. It also left out the couple's SSI income."
+us,scenario_079,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said $630 a month of SSDI was above the income limit. Disabled households skip the gross income test, and even with SSI included, net income of $1,157 is well below the $1,762.50 net limit (100% FPL), so the household qualifies for $198 a month."
+us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the couple's $881 a month of SSI. It also used a $219 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction, then arbitrarily lowered its own $420.50 result to $351.50."
+us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Treated SSDI ($630 a month) as the only income and never added the $881 a month of SSI the disabled couple receives. On top of that, it guessed a net income of 'near $300' instead of applying $209 plus Arizona's $145 standard medical deduction to $1,511."
+us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Left out the couple's $881 a month of SSI and used only $630 of SSDI as income. It also misapplied the medical deduction, treating $632 of annual expenses as a monthly figure, instead of applying Arizona's $145 standard medical deduction."
+us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"Got the $1,511 income and $209 standard deduction right but deducted only the $26 actual medical excess, missing Arizona's $145 standard medical deduction for elderly/disabled households. Net income came out $119 too high, and the benefit $163 a month instead of $198."
+us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Said the household receives SSI/SSDI but counted only the $630 of SSDI and never added the $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, along with outdated $204 and $535 figures."
+us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Made up large shelter costs from the mortgage balance even though no mortgage payment or interest was given, and left out the $881 a month of SSI. It then reported $374 a month as the 'maximum allotment', which contradicts the actual $546 two-person maximum."
+us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the couple's $881 a month of SSI, so it used net income of $395 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $228 standard deduction, a $549 maximum, and $876 of SSI instead of $209, $546, and $881."
+us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Counted vehicle equity above a $4,650 exclusion as a resource. Under Arizona's broad-based categorical eligibility (and PolicyEngine's asset test), vehicles are not counted, so countable assets are just the $1,045.29 bank balance and the household passes."
+us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also used a $202 standard deduction, a $548 maximum, and SSI of $872 a month instead of $209, $546, and $881."
+us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Used only $630 of SSDI as income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, which gave it a $427 monthly benefit instead of $198."
+us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also understated SSI at $805 instead of $881 and used outdated $198 standard-deduction and $535 maximum figures.
+us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Treated having no rent or utility costs as making the benefit zero. Shelter costs only feed an optional deduction and are not required for eligibility; with $1,157 of net income the household receives $546 − $348 = $198 a month."
+us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Claimed 30% of net income exceeds the maximum allotment. Net income after the $209 standard deduction and Arizona's $145 medical deduction is $1,157, so the household's contribution is only $348 against a $546 maximum, giving a regular $198 benefit rather than the minimum benefit."
+us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,Deducted only $312 a year ($26 a month) of medical expense instead of Arizona's $145-a-month standard medical deduction. It also used a $200 standard deduction and a $560 maximum allotment instead of $209 and $546.
+us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"Declared the household ineligible on asset and income grounds without any computation. Countable assets are only $1,045.29 and net income of $1,157 is well below the net limit, so the household receives $198 a month."
+us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $178 monthly benefit matches deducting only the actual $26 medical excess instead of Arizona's $145 standard medical deduction. With the $145 deduction, net income is $1,157 and the benefit is $198 a month."
+us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $470 monthly benefit matches counting only the $630 of SSDI and leaving out the couple's $881 a month of SSI. Including SSI, net income is $1,157 and the benefit is $198."
+us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Counted only the $432 of other medical expenses, dropped the $300 of over-the-counter costs, and deducted $12 a year instead of Arizona's $145-a-month standard medical deduction. It also used SSI of $840, a $204 standard deduction, and a $536 maximum instead of current values."
+us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"Did not return a usable SNAP value. The correct derivation is $546 − 30% of $1,157 net income = $198 a month, or $2,376 a year."
+us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used only the $7,560 of SSDI as gross income and left out the couple's $10,572 a year of SSI. It also deducted the $312 actual medical excess instead of Arizona's $1,740-a-year ($145-a-month) standard medical deduction."
+us,scenario_079,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed SNAP is zero without dependents or shelter costs. Neither is required: the disabled couple has $1,157 of net income, below the $1,762.50 limit, and receives $198 a month."
+us,scenario_079,snap,gpt-5.4-nano,llm_error,other,False,"Did no SNAP calculation and defaulted to zero, even though the facts are enough: $1,511 of SSDI+SSI income, the $209 standard deduction, and Arizona's $145 medical deduction give $198 a month."
+us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and used a $204 standard deduction instead of $209. That left net income at $1,281 instead of $1,157 and the benefit at $161 instead of $198."
+us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Awarded the full $546 maximum without subtracting 30% of net income. The household's $1,157 of net income produces a $348 contribution, so the benefit is $198 a month."
+us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163 monthly benefit matches deducting the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households, which overstated net income by $119."
+us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $165 monthly benefit matches deducting only the actual $26 medical excess instead of Arizona's $145 standard medical deduction. With $145, net income is $1,157 and the benefit is $198."
+us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"Deducted only $26 of excess medical expense instead of Arizona's $145 standard medical deduction, giving net income of $1,276 instead of $1,157. It also skipped rounding the 30% contribution up to a whole dollar."
+us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,Its $395 net income comes from counting only the $630 of SSDI and leaving out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
+us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"Got the $1,511 income and $209 standard deduction right but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction, which gave $163 a month instead of $198."
+us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Deducted the $26 actual medical excess instead of Arizona's $145 standard medical deduction, giving net income of $1,276 instead of $1,157 and an annual benefit of $1,958.40 instead of $2,376."
+us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"Claimed SSDI income exceeds the SNAP limits without applying any test. Disabled households face only the net income test, and $1,157 of net income is far below the $1,762.50 limit, so the benefit is $198 a month."
+us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used only the $7,560 of SSDI as income and left out the couple's $881 a month of SSI. It also deducted $312 a year of actual medical excess instead of Arizona's $145-a-month standard medical deduction."
+us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used $876 of SSI and a $549 maximum instead of $881 and $546.
+us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"Got the $1,511 income, $209 standard deduction, and $546 maximum right but deducted only $26 of excess medical expense instead of Arizona's $145 standard medical deduction, which gave $163.20 a month instead of $198."
+us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"Deducted the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used the 2025 SSI rate ($840), a $204 standard deduction, and a $536 maximum instead of the 2026 values of $881, $209, and $546."
+us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,Used only the $630 of monthly SSDI as gross income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
+us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return a usable SNAP value. The correct derivation is $546 − 30% of $1,157 net income = $198 a month, or $2,376 a year."
+us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the $881 a month of SSI the disabled couple receives, so net income was $395 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,minimax-m3,llm_error,other,False,"Admitted income is below the thresholds but returned zero to avoid 'inferring benefit receipt', which contradicts the prompt's instruction to assume program take-up. The eligible household receives $198 a month."
+us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,Treated the $630 of SSDI as the household's only income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
+us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Left out the couple's SSI income and applied the $35 medical threshold annually instead of monthly. It also used a $598 maximum and a $198 standard deduction instead of $546 and $209, and missed Arizona's $145 standard medical deduction."
+us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Applied a California-style SSI cash-out rule that bars SSI recipients from SNAP. Arizona has no such exclusion, and SSI receipt instead confers categorical eligibility, so the couple receives $198 a month."
us,scenario_079,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that the spouse's ""income and circumstances"" fail Arizona's adult Medicaid rules, inverting the test: the household's MAGI is 0.35 × FPL, far below Arizona's 138% FPL expansion-adult limit, so low income qualifies rather than disqualifies. It also skipped the pathway the engine used — the SSI-recipient category, which grants automatic Medicaid in Arizona (a 1634 state) to the blind/disabled spouse whose $0 countable income and $1,045 in resources (under the $2,000 individual limit) produce $5,286 of SSI — and treated an income screen as the sole route to adult coverage."
us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to 0 on the ground that ""no additional Medicaid-eligibility inputs"" beyond disability and IHS coverage were supplied, treating Medicaid eligibility as requiring an explicit enrollment or income input rather than deriving it from the facts given. Blindness plus disability plus $0 earned and unearned income and $1,045 in countable resources produce SSI of $5,286, and SSI receipt in Arizona triggers the SSI-recipient category's automatic Medicaid; separately, 0.35 × FPL MAGI clears the 138% FPL expansion-adult test, and IHS coverage at interview is not minimum essential coverage that blocks eligibility."
us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disabled flag as enough for Medicare under 65, citing 'SSDI-linked disability eligibility'. However, the spouse gets no Social Security disability income; the $7,560 SSDI is the head's. No months of SSDI receipt are listed, so the 24-month SSDI entitlement requirement is not met and the spouse is not Medicare eligible."
us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model correctly stated that disability-based Medicare requires 24 months of SSDI receipt, then set that requirement aside and marked the spouse eligible for being disabled. It also wrongly gave the spouse 'SSDI-equivalent status' when only the head has Social Security disability income. With zero SSDI and zero months of receipt, the spouse fails the 24-month test and, at 55, the age-65 test."
us,scenario_079,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model cited a 'disability-based Medicare eligibility rule' but applied it to the disabled flag alone. The disability pathway requires 24 months of SSDI entitlement (or ESRD/ALS). The spouse has no SSDI income, no months of receipt, and no ESRD, and at 55 is under 65, so the correct answer is not eligible."
us,scenario_079,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model claimed the spouse qualifies under disability-based Medicare rules, which equates being disabled with Medicare entitlement. Those rules require at least 24 months of SSDI receipt or ESRD/ALS. The spouse receives no Social Security disability benefits (the SSDI belongs to the head) and is 55, so the spouse is not eligible."
-us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"It computed countable income correctly ($630 SSDI - $20 = $610/mo). For the couple rate it used a projected $1,450.60/mo, which is essentially the 2025 couple FBR, instead of the 2026 couple FBR of $1,491/mo. As a result, 12 x ($1,450.60 - $610) = $10,088.40 understates the correct 12 x $881 = $10,572."
-us,scenario_079,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied an individual income-limit test and claimed that $7,560/yr of SSDI exceeds a $11,316/yr limit, which is false on its own numbers. It also ignored that both spouses are blind and disabled and form an eligible couple under the $1,491/mo couple FBR. Countable income is only $610/mo, which leaves an $881/mo benefit, not $0."
-us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"It first computed $10,080 using a $1,450/mo couple FBR. It then replaced that with an invented $1,940/mo couple FBR ($23,280/yr), producing $15,960. The 2026 couple FBR is $1,491/mo, so the correct benefit is ($1,491 - $610) x 12 = $10,572."
-us,scenario_079,ssi,claude-opus-4.8,llm_error,thresholds_rates,False,"It overstated the 2026 couple FBR at $1,538/mo (actual: $1,491) and derived $11,136 in its explanation. It then submitted $7,212, which matches neither its own derivation nor the rules. Against its $7,320 of countable income, that figure implies a couple FBR of only about $1,211/mo. The correct result is ($1,491 - $610) x 12 = $10,572."
-us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"It set up the calculation almost correctly: a $1,489/mo couple FBR ($17,868/yr) minus $7,320 of countable income. It then subtracted wrong: $17,868 - $7,320 is $10,548, not the $6,072 it submitted. With the correct 2026 couple FBR of $1,491/mo, the answer is $10,572."
-us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took $1,415/mo as the 2025 couple FBR (that is the 2024 rate) and inflated it 2.5% to $1,450/mo, which is the 2025 rate. The 2026 couple FBR is $1,491/mo. With the correct $610 of countable income, it got $840/mo instead of $881/mo, or $10,080 instead of $10,572."
-us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"Its explanation finds the couple eligible and computes $15,804/yr using an inflated $1,927/mo couple FBR, yet it submitted 0, which contradicts its own conclusion. The 2026 couple FBR is $1,491/mo, so the correct benefit is ($1,491 - $610) x 12 = $10,572."
-us,scenario_079,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The method is correct: couple FBR minus $610 of countable SSDI. It used a $1,486/mo couple FBR instead of the 2026 value of $1,491/mo, so it got $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly applied the couple computation with $610/mo of countable income. It used a $1,485/mo couple FBR instead of the 2026 value of $1,491/mo, which gives $875/mo ($10,500/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly subtracted $610/mo of countable SSDI from the couple rate. The rate it used was $1,482/mo instead of the 2026 couple FBR of $1,491/mo, so it got $872/mo ($10,464/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly computed $610/mo of countable income. It used a $1,486.25/mo couple FBR instead of the 2026 value of $1,491/mo, which yields $876.25/mo ($10,515/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used $1,415/mo as the couple SSI rate, which is the 2024 couple FBR, instead of the 2026 value of $1,491/mo. With the correct $610 of countable income, that gives $805/mo ($9,660/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It asserted that the SSDI exceeds the couple federal benefit rate. In fact, countable SSDI is $610/mo after the $20 exclusion, far below the 2026 couple FBR of $1,491/mo. The couple is therefore eligible for $881/mo, or $10,572/yr, not $0."
-us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly computed $7,320 of annual countable income. It subtracted that from a $16,980 couple maximum, which is the 2024 rate of $1,415/mo. The 2026 couple FBR is $17,892/yr ($1,491/mo), which gives $10,572."
-us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a projected couple FBR of $17,832/yr ($1,486/mo) instead of the 2026 value of $17,892/yr ($1,491/mo). Subtracting the correct $7,320 of countable income gave $10,512 instead of $10,572."
-us,scenario_079,ssi,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave $0 without any computation. Both spouses are blind and disabled, so they form an eligible couple. Countable resources of $1,045 are under the $3,000 couple limit, and countable income of $610/mo is under the $1,491/mo couple FBR, which yields $881/mo, or $10,572/yr."
-us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly subtracted $7,320 of countable SSDI. Its $10,536 answer implies a couple FBR of $17,856/yr ($1,488/mo) instead of the 2026 value of $17,892/yr ($1,491/mo), which gives $10,572."
-us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the $20/mo exclusion correctly, leaving $7,320/yr of countable income. Its $10,488 answer implies a couple FBR of $17,808/yr ($1,484/mo) instead of the 2026 value of $1,491/mo, which gives $10,572."
-us,scenario_079,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a couple FBR of $1,450/mo ($17,400/yr), which is the 2025 rate, instead of the 2026 rate of $1,491/mo ($17,892/yr). Subtracting the correct $7,320 of countable income gave $10,080 instead of $10,572."
-us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"It returned no SSI value and no explanation, so no answer was scored. The correct derivation is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, times 12, which is $10,572."
-us,scenario_079,ssi,glm-5.3,llm_error,thresholds_rates,False,"It correctly derived $610/mo of countable income. It used an approximate couple FBR of $1,490/mo instead of the 2026 value of $1,491/mo, so it got $880/mo ($10,560/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"It counted the vehicle toward the resource limit and concluded the couple failed the resource test. SSI excludes one automobile regardless of value, so countable resources are only the $1,045 bank account, well under the $3,000 couple limit. The couple is eligible for ($1,491 - $610) x 12 = $10,572."
-us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 because no SSI receipt was listed, instead of computing eligibility with take-up assumed, as the prompt directs. Blindness and disability make both spouses an eligible couple. The benefit is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, or $10,572/yr."
-us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly reduced the couple rate by $610 of countable SSDI. It used a projected $1,487/mo couple maximum instead of the 2026 value of $1,491/mo, which gives $877/mo ($10,524/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"It correctly computed $610/mo of countable income. It used a $1,486/mo couple payment standard instead of the 2026 couple FBR of $1,491/mo, which gives $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"It asserted that the SSDI exceeds the couple's SSI limits without computing anything. Countable SSDI is $630 - $20 = $610/mo, which is well below the 2026 couple FBR of $1,491/mo. That leaves an $881/mo benefit, or $10,572/yr, not $0."
-us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"It correctly took $610/mo of countable SSDI. It used a couple FBR of about $1,486/mo instead of the 2026 value of $1,491/mo, so it got $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,grok-4.6,llm_error,thresholds_rates,False,"Its resource and income analysis is correct ($1,045 of countable resources and $610/mo of countable income). It used a couple FBR of about $1,486/mo instead of the 2026 value of $1,491/mo, which gives $10,512 instead of $10,572."
-us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly applied the 2025 couple SSI rate of $1,450/mo to a 2026 case instead of the 2026 couple FBR of $1,491/mo. With the correct $610 of countable income, it got $840/mo ($10,080/yr) instead of $881/mo ($10,572/yr)."
-us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SSI value and no explanation, so no answer was scored. The correct derivation is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, times 12, which is $10,572."
-us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"It read the rule against inferring unlisted benefit receipt as a reason to report $0 SSI. The prompt says to assume program take-up, and SSI is the output to compute. The blind and disabled couple is eligible for the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, or $10,572/yr."
-us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"It estimated the 2026 couple FBR at $1,450/mo ($17,400/yr), which is the 2025 rate, instead of the 2026 rate of $1,491/mo ($17,892/yr). Subtracting the correct $7,320 of countable income gave $10,080 instead of $10,572."
-us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It treated each spouse as an individual claimant and summed individual rates instead of applying the $1,491/mo couple FBR. It also counted only $3,780 of combined income, which is half the SSDI. Under couple computation, the full $7,560 counts against the couple ($610/mo after the $20 exclusion), which leaves $881/mo, or $10,572/yr, not $18,963.60."
-us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said the increased excise tax credit requires filing with income and wrote the couple off because they have only nontaxable SSDI. The credit is refundable and goes to filers with federal AGI at or below $12,500 (MFJ), so this couple's $0 AGI qualifies them for $25 x 2 = $50."
-us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Arizona has no refundable individual income tax credit and treated the absence of taxable income as disqualifying. It missed the refundable increased excise tax credit, which is keyed to low federal AGI rather than taxable income and pays $25 each to head and spouse, $50 in total."
-us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly worked out the increased excise tax credit at $25 per person, $50 for an MFJ couple with no dependents, and ruled out the property tax credit. It then submitted $200 with no computation behind it. That figure contradicts the model's own $50 derivation and even exceeds the credit's $100 cap."
-us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model claimed the increased excise tax credit requires meeting income tax filing thresholds, and it tied refundable credits to taxable wages. The credit is refundable and is designed for filers whose federal AGI is below $12,500 (MFJ), so this $0-AGI couple receives $25 x 2 = $50."
-us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model conditioned Arizona refundable credits on earnings or qualifying dependents. The increased excise tax credit requires neither: it pays $25 for each of head and spouse ($50) to an MFJ unit with federal AGI at or below $12,500."
-us,scenario_079,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model concluded that no Arizona refundable credit applies and overlooked the increased excise tax credit. That credit pays $25 per taxpayer, $50 for head and spouse, because the couple's federal AGI of $0 is below the $12,500 MFJ ceiling."
-us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model labeled the increased excise tax credit nonrefundable and required Arizona taxable income or wages for any refundable credit. The credit is refundable and depends only on federal AGI of $12,500 or less for MFJ filers, which yields $25 x 2 = $50 here."
-us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model checked only the Property Tax Credit (Form 140PTC) and then dismissed every other refundable credit. It missed the increased excise tax credit, which pays $25 per person ($50) to an MFJ household with federal AGI under $12,500 regardless of age or SSI status."
-us,scenario_079,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model treated the absence of tax liability as ruling out a refundable credit. Arizona's increased excise tax credit is fully refundable and pays this zero-liability, zero-AGI MFJ couple $25 each, $50 in total."
-us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model treated zero income and no qualifying children as disqualifying. Zero federal AGI is exactly what qualifies the couple for the increased excise tax credit, which counts head and spouse at $25 each, $50 in total, under the $12,500 MFJ AGI ceiling."
-us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model looked only for child-based and property tax credits. It missed the increased excise tax credit, which is refundable, counts the filer and spouse at $25 each and pays $50 because federal AGI is under $12,500."
-us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model asserted that no Arizona refundable credit applies without checking the increased excise tax credit. That credit pays $25 per taxpayer, $50 for this MFJ couple, because their federal AGI of $0 is under the $12,500 limit."
-us,scenario_079,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model made refundable credits depend on having state income tax liability. The increased excise tax credit is refundable regardless of liability and pays $25 x 2 = $50 to an MFJ unit with federal AGI at or below $12,500."
-us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model concluded from the low income and demographics that no state refundable credit applies. Low income is precisely the trigger for Arizona's refundable increased excise tax credit, which pays $25 for each of head and spouse, $50 in total, at federal AGI of $12,500 or less."
-us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model stated that the household qualifies for no state refundable credit and never applied the increased excise tax credit. That credit gives $25 per filer to MFJ units with federal AGI of $12,500 or less, which is $50 for this $0-AGI couple."
-us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stated that no refundable Arizona credit applies and omitted the increased excise tax credit. That credit is refundable, is available at federal AGI of $12,500 or less for MFJ filers and pays $25 x 2 = $50 here."
-us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model required state income tax liability or qualifying earnings for a refundable credit. The increased excise tax credit needs neither: it pays $25 each to head and spouse, $50 in total, because federal AGI is $0, under the $12,500 MFJ limit."
-us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model concluded that the household qualifies for no refundable Arizona credit and missed the increased excise tax credit. That credit pays $25 per person, $50 for this MFJ couple, because their federal AGI is $0, below the $12,500 ceiling."
-us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model denied any refundable Arizona credit and never applied the increased excise tax credit. That credit gives $25 for each of head and spouse ($50) to MFJ filers with federal AGI of $12,500 or less."
-us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model asserted that childless married couples without wages get no refundable Arizona credits. The increased excise tax credit pays such couples $25 per spouse, $50 in total, when federal AGI is at or below $12,500, with no wage or child requirement."
-us,scenario_079,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated zero taxable income as ruling out Arizona refundable credits. The increased excise tax credit is refundable, turns on federal AGI of $12,500 or less (MFJ) rather than taxable income, and pays $25 x 2 = $50 here."
-us,scenario_079,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model considered only a family credit tied to the federal EITC and asserted that Arizona has no other refundable credits. It missed the increased excise tax credit, which pays $25 each to head and spouse, $50 in total, because federal AGI is $0, under the $12,500 MFJ limit."
-us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no refundable Arizona credit triggered by the facts and overlooked the increased excise tax credit. That credit is triggered by federal AGI of $12,500 or less for MFJ filers and pays $25 per spouse, $50 in total."
-us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model required earned income or children for a state refundable credit. Arizona's increased excise tax credit requires neither: it pays $25 for each of head and spouse ($50) to MFJ filers with federal AGI of $12,500 or less."
-us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model conditioned refundable Arizona credits on tax liability, earned income or qualifying children. The increased excise tax credit is refundable without any of these and pays $25 x 2 = $50 because federal AGI is under $12,500 (MFJ)."
-us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded that no reported fact generates an Arizona refundable credit. The facts do generate one: federal AGI of $0 with MFJ status and two adults qualifies the couple for the increased excise tax credit of $25 per person, $50 in total."
-us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model required qualifying earnings or children for an Arizona refundable credit. The increased excise tax credit pays $25 to each of head and spouse, $50 in total, to MFJ filers with federal AGI at or below $12,500, without earnings or children."
-us,scenario_079,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model required taxable income or dependents for a refundable Arizona credit. The increased excise tax credit is keyed to federal AGI of $12,500 or less (MFJ) and counts the filer and spouse at $25 each, $50 in total."
-us,scenario_079,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,The model conditioned Arizona refundable credits on earnings or dependent children. The increased excise tax credit needs neither and pays this $0-AGI MFJ couple $25 x 2 = $50.
-us,scenario_079,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model treated the absence of state tax liability as eliminating refundable credits. The increased excise tax credit is refundable regardless of liability and pays $50 ($25 per spouse) to MFJ filers with federal AGI of $12,500 or less."
-us,scenario_079,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model looked for qualifying children and tax liability as the basis for Arizona refundable credits. The increased excise tax credit is based on per-person counts of filer and spouse at $25 each, gated only by federal AGI of $12,500 or less (MFJ), and yields $50."
-us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model assumed Arizona has a state EITC set at a percentage of the federal EITC, which Arizona does not have, and then declared that no other refundable credit applies. It missed the increased excise tax credit, which pays $25 x 2 = $50 because federal AGI is $0, under the $12,500 MFJ limit."
-us,scenario_079,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model tied refundable credits to state tax liability and found none applicable. The increased excise tax credit is refundable and pays this zero-liability MFJ couple $25 each, $50 in total, because federal AGI is under $12,500."
-us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked only the property tax refund, which needs rent or property tax, and assumed every other refundable credit needs earned income. The increased excise tax credit needs neither and pays $25 for each of head and spouse ($50) at federal AGI of $12,500 or less (MFJ)."
-us,scenario_079,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model treated being childless with zero liability as disqualifying. The increased excise tax credit counts the filer and spouse themselves at $25 each, $50 in total, and is refundable at federal AGI of $12,500 or less for MFJ filers."
-us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model asserted that Arizona has no state income tax. Arizona levies a flat individual income tax and offers the refundable increased excise tax credit, which pays this $0-AGI MFJ couple $25 x 2 = $50."
-us,scenario_079,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model correctly ruled out the property tax credit on its age-65 test, then assumed no other refundable credit applies at $0 liability. It missed the increased excise tax credit, which has no age test and pays $25 per spouse, $50 in total, to MFJ filers with federal AGI of $12,500 or less."
-us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model required Arizona taxable income or qualifying dependents for a refundable credit. The increased excise tax credit is gated by federal AGI of $12,500 or less (MFJ), counts head and spouse at $25 each and yields $50."
-us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model asserted that no Arizona refundable credit applies and never applied the increased excise tax credit. That credit pays $25 for each of head and spouse ($50) because the couple's federal AGI is $0, below the $12,500 MFJ ceiling."
+us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"It used a couple federal benefit rate of about $1,450.60/month, which is effectively the 2025 couple rate, instead of the 2026 couple rate of $1,491/month. With the same $610 countable income, that gives $840.60/month instead of $881/month, so it understated SSI by about $484/year."
+us,scenario_079,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It wrongly said $7,560/year of SSDI exceeds a $11,316/year individual limit, even though $7,560 is less than $11,316. It also never applied the couple test, where $610/month of countable income (after the $20 exclusion) is well below the $1,491/month couple rate, leaving $881/month payable."
+us,scenario_079,ssi,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said the head's SSDI exceeds the couple income limit, but $630/month minus the $20 general exclusion is $610/month. That is far below the 2026 couple rate of $1,491/month, so the couple is owed $881/month ($10,572/year), not zero."
+us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"It first used $1,450/month and then switched to an inflated couple rate of $1,940/month. The correct 2026 couple rate is $1,491/month, so $1,491 − $610 = $881/month ($10,572/year); its $23,280 − $7,320 = $15,960 overstates the benefit."
+us,scenario_079,ssi,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a couple rate of $1,538/month instead of the 2026 rate of $1,491/month, which produced $11,136. It then submitted $7,212, which matches none of its own calculations. The correct result is $881/month × 12 = $10,572."
+us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"It made a subtraction error: $17,868 − $7,320 is $10,548, not the $6,072 it submitted. Its couple rate of $1,489/month is also $2 below the 2026 rate of $1,491/month, which would have given $10,572."
+us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It started from $1,415 as the 2025 couple rate (that is the 2024 rate) and projected forward to $1,450/month, which is actually the 2025 rate. The 2026 couple rate is $1,491/month, so the benefit is $881/month, not $840/month."
+us,scenario_079,ssi,claude-sonnet-5,llm_error,thresholds_rates,False,"It used an inflated couple rate of $1,927/month and calculated $15,804/year. It then submitted 0, which contradicts its own conclusion that the couple is eligible. The correct calculation is $1,491 − $610 = $881/month ($10,572/year)."
+us,scenario_079,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a couple rate of $1,486/month instead of the 2026 rate of $1,491/month. That gives $876/month instead of $881/month, so it was $60/year low."
+us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a couple rate of $1,485/month instead of the 2026 rate of $1,491/month. That gives $875/month instead of $881/month, so it was $72/year low."
+us,scenario_079,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a couple rate of $1,482/month instead of the 2026 rate of $1,491/month. That gives $872/month instead of $881/month, so it was $108/year low."
+us,scenario_079,ssi,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a couple rate of $1,486.25/month instead of the 2026 rate of $1,491/month. That gives $876.25/month instead of $881/month, so it got $10,515 instead of $10,572."
+us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the 2024 couple rate of $1,415/month instead of the 2026 rate of $1,491/month. That gives $805/month instead of $881/month, so it understated SSI by $912/year."
+us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said the SSDI exceeds the couple rate, but countable income is $610/month ($630 minus the $20 exclusion). That is well below the 2026 couple rate of $1,491/month, leaving $881/month payable."
+us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a maximum couple benefit of $16,980/year ($1,415/month, the 2024 rate) instead of the 2026 rate of $17,892/year ($1,491/month). That gave $9,660 instead of $10,572."
+us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"It projected the 2026 couple rate as $17,832/year ($1,486/month) instead of the actual $17,892/year ($1,491/month). Subtracting $7,320 of countable income therefore gave $10,512 instead of $10,572."
+us,scenario_079,ssi,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned zero with no derivation, which implies it treated this blind and disabled couple as ineligible. The couple passes the resource test ($1,045 against a $3,000 limit) and has only $610/month of countable income against the $1,491/month couple rate, so the benefit is $881/month."
+us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $10,536 implies a couple rate of $1,488/month instead of the 2026 rate of $1,491/month. With the same $610/month countable income, that understates the annual benefit by $36."
+us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $10,488 implies a couple rate of $1,484/month instead of the 2026 rate of $1,491/month. That gives $874/month instead of $881/month."
+us,scenario_079,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 couple rate of $17,400/year ($1,450/month) instead of the 2026 rate of $17,892/year ($1,491/month). That gave $10,080 instead of $10,572."
+us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for SSI, so there was no answer to score against the $10,572 benefit."
+us,scenario_079,ssi,glm-5.3,llm_error,thresholds_rates,False,"It used a couple rate of about $1,490/month instead of the 2026 rate of $1,491/month. That gives $880/month instead of $881/month, so it was $12/year low."
+us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"It counted the vehicle and the bank account against the resource limit, but SSI excludes one vehicle regardless of value. The only countable resource is the $1,045 bank account, which is under the $3,000 couple limit, so the couple qualifies for $881/month."
+us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It treated SSI as an amount that had to be listed as received rather than computing eligibility, even though the prompt says to assume take-up. Both spouses are blind and disabled, which makes them categorically eligible, and the benefit is $1,491 − $610 = $881/month."
+us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"It projected the 2026 couple rate as $1,487/month instead of $1,491/month. That gives $877/month instead of $881/month, so it got $10,524 instead of $10,572."
+us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a couple rate of $1,486/month instead of the 2026 rate of $1,491/month. That gives $876/month instead of $881/month."
+us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"It assumed the SSDI pushes the couple over the income limit without checking it. The $610/month of countable income (after the $20 exclusion) is far below the 2026 couple rate of $1,491/month, leaving an $881/month benefit."
+us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"It used a couple rate of about $1,486/month instead of the 2026 rate of $1,491/month. That gives $876/month instead of $881/month, so it was $60/year low."
+us,scenario_079,ssi,grok-4.6,llm_error,thresholds_rates,False,"It used a couple rate of about $1,486/month instead of the 2026 rate of $1,491/month. Its eligibility, resource, and income steps were correct, but the rate error gave $876/month instead of $881/month."
+us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly used the 2025 couple rate of $1,450/month for a 2026 tax year instead of the 2026 rate of $1,491/month. That gave $840/month instead of $881/month."
+us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for SSI, so there was no answer to score against the $10,572 benefit."
+us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"It misread the instruction not to infer benefit receipt as a reason to skip SSI, even though the prompt says to assume program take-up. Computing eligibility gives the blind and disabled couple $1,491 − $610 = $881/month."
+us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"It used $1,450/month ($17,400/year) as the 2026 couple rate, but $1,450 is the 2025 rate. The 2026 rate is $1,491/month ($17,892/year), so the benefit is $17,892 − $7,320 = $10,572, not $10,080."
+us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It combined two individual rates (about $947.65/month each) instead of using the single 2026 couple rate of $1,491/month. It also counted only $3,780 of the $7,560 SSDI and skipped the $20 exclusion, when the couple's countable income is $7,320/year. Together these errors inflated SSI to $18,963.60."
+us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model said the Increased Excise Tax Credit 'requires filing with income' and needs earned income. In fact the credit is refundable and only requires federal AGI of $25,000 or less for MFJ; zero income qualifies. It should have given $25 × 2 exemptions = $50 instead of $0."
+us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model wrongly said Arizona has no refundable individual income tax credit, and it treated having no taxable income as disqualifying. The refundable Increased Excise Tax Credit pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, so this couple with $0 AGI gets $50."
+us,scenario_079,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. It pays $25 per personal exemption to MFJ filers with federal AGI of $25,000 or less, and this couple has $0 AGI, so the credit is $50."
+us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly worked out the Increased Excise Tax Credit as $25 per person, giving $50 for an MFJ couple with no dependents. It then threw that result away and submitted an unsupported 'best estimate' of $200. The correct total is $50."
+us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model claimed the Increased Excise Tax Credit 'requires income tax filing thresholds not met here.' The credit has an income ceiling (federal AGI of $25,000 or less for MFJ), not a minimum income or filing threshold. With $0 AGI the couple qualifies for $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed AZ refundable credits require earnings or qualifying dependents. The Increased Excise Tax Credit pays $25 for each personal exemption to low-AGI filers without either, so this couple gets $50."
+us,scenario_079,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model overlooked Arizona's refundable Increased Excise Tax Credit, which is $25 per exemption for MFJ filers with federal AGI of $25,000 or less. This couple's AGI is $0, so the credit is $50."
+us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly grouped the Increased Excise Tax Credit with nonrefundable credits and said it requires wages or AZ taxable income. The credit is refundable and only requires federal AGI of $25,000 or less for MFJ, so the couple gets $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model only checked the age-65/SSI-gated Property Tax Credit and missed the refundable Increased Excise Tax Credit. That credit has no age or SSI requirement and pays this $0-AGI MFJ couple $25 per exemption, or $50."
+us,scenario_079,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model treated having no tax liability as ruling out refundable credits. Arizona's Increased Excise Tax Credit is refunded without any liability, at $25 per exemption for MFJ filers with AGI of $25,000 or less, giving $50 here."
+us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model assumed refundable credits need income or qualifying children. Arizona's Increased Excise Tax Credit is paid per personal exemption precisely to zero- and low-AGI filers, so this couple gets $50."
+us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model checked dependent-child and property-tax credits but missed the refundable Increased Excise Tax Credit. That credit pays $25 for each spouse's exemption when MFJ federal AGI is $25,000 or less, for a total of $50."
+us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. This couple has $0 AGI, so the credit is $50."
+us,scenario_079,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,The model treated zero state tax liability as ruling out refundable credits. The Increased Excise Tax Credit is refundable without any liability and pays $25 per exemption for a total of $50.
+us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. Its only test is federal AGI of $25,000 or less for MFJ, which this couple meets at $0 AGI, so it pays $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. The couple qualifies through $0 federal AGI, well under the $25,000 MFJ cap, and receives $25 per exemption, or $50."
+us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. That gives $50 for this couple with $0 AGI."
+us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model assumed refundable credits require tax liability or earnings. Arizona's Increased Excise Tax Credit requires neither and pays $25 per personal exemption to low-AGI filers, so this couple gets $50."
+us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. The MFJ couple's $0 federal AGI is under the $25,000 cap, so the credit is $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. That gives $50 here."
+us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model claimed that under PolicyEngine rules, childless couples with no wages get no AZ refundable credits. In fact the Increased Excise Tax Credit requires no wages or children and pays $25 per personal exemption, giving $50 for this $0-AGI MFJ couple."
+us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model treated zero taxable income as ruling out AZ refundable credits. The Increased Excise Tax Credit is refundable regardless of taxable income and pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, for $50 here."
+us,scenario_079,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"The model wrongly said Arizona's only refundable credit depends on federal EITC. Arizona has no EITC-linked credit; its refundable Increased Excise Tax Credit pays $25 per exemption to this $0-AGI MFJ couple, for $50."
+us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model missed that the listed facts (Arizona residence, MFJ filing, two exemptions, $0 federal AGI) are enough for the refundable Increased Excise Tax Credit, which comes to $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required earned income or children for an AZ refundable credit. The Increased Excise Tax Credit is based only on personal exemptions and an AGI ceiling of $25,000 for MFJ, so this couple gets $50."
+us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model assumed an AZ refundable credit needs liability, earned income, or children. The Increased Excise Tax Credit needs none of these and pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, for $50."
+us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model missed that Arizona residency plus $0 federal AGI on an MFJ return qualifies for the refundable Increased Excise Tax Credit, which pays $25 for each spouse, or $50."
+us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model required earnings or children for an AZ refundable credit. The Increased Excise Tax Credit pays $25 per personal exemption to low-AGI filers without either, giving $50 for this couple."
+us,scenario_079,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"The model treated having no taxable income or dependents as ruling out AZ refundable credits. The Increased Excise Tax Credit is refundable at $25 per personal exemption when MFJ federal AGI is $25,000 or less, giving $50."
+us,scenario_079,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"The model required earnings or dependent children. Arizona's refundable Increased Excise Tax Credit pays $25 per personal exemption to MFJ filers with federal AGI of $25,000 or less, so this couple gets $50."
+us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model treated zero state tax liability as ruling out refundable credits. Arizona's refundable Increased Excise Tax Credit pays $25 per exemption regardless of liability, for $50 here."
+us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model assumed AZ refundable credits need children and tax liability. The Increased Excise Tax Credit is paid per personal exemption to filers with AGI of $25,000 or less, giving $50 for this couple."
+us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model invented an Arizona state EITC tied to the federal EITC; Arizona has none. It also missed the refundable Increased Excise Tax Credit, which pays this $0-AGI MFJ couple $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model treated zero state tax liability as ruling out refundable credits. Arizona's Increased Excise Tax Credit is refunded in full at $25 per exemption to MFJ filers with AGI of $25,000 or less, giving $50."
+us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model only checked the property-tax credit and earned-income-based credits, and missed the refundable Increased Excise Tax Credit. That credit needs no rent, property tax, or earnings and pays this $0-AGI MFJ couple $50."
+us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model said no AZ refundable credit applies to a childless household with no liability. The Increased Excise Tax Credit pays $25 per personal exemption to exactly such low-AGI filers, so this couple gets $50."
+us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model wrongly claimed Arizona has no state income tax; Arizona levies a 2.5% flat income tax. Its income tax system includes the refundable Increased Excise Tax Credit, which pays this couple $25 × 2 = $50."
+us,scenario_079,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model correctly ruled out the age-65 Property Tax Credit but missed the refundable Increased Excise Tax Credit. That credit has no age test and pays $25 per exemption when MFJ federal AGI is $25,000 or less, giving $50."
+us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated having no AZ taxable income or dependents as disqualifying. The refundable Increased Excise Tax Credit is $25 per personal exemption for MFJ filers with federal AGI of $25,000 or less, so this couple gets $50."
+us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. This MFJ couple's $0 federal AGI qualifies them for $25 per exemption, a total of $50."
us,scenario_080,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_080,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_080,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_080,head_chip_eligible,claude-sonnet-5.5,llm_error,other,False,"Its explanation says the head is not eligible and ends with 'value = 0', which matches the correct result, but its scored answer records Yes. The submission breaks the contract that the numeric value must match the explanation. Its reasoning relied on age alone (CHIP is for children). The rule the engine applies is that the head is Medicaid-eligible under the PA adult expansion category, and that excludes CHIP."
us,scenario_080,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"GLM gave no value and no explanation for head_chip_eligible, so there was nothing to score. The correct derivation is that the 41-year-old head, with about $940 of income, is Medicaid-eligible under Pennsylvania's ACA adult expansion category. That excludes her from CHIP, so the answer is 0."
-us,scenario_080,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Because the head is disabled, the model routed him into the aged/blind/disabled pathway and demanded SSI-linked categorical status plus an asset screen, but Pennsylvania adopted the ACA adult expansion, whose MAGI group covers every non-elderly adult 19-64 at or below 138% FPL regardless of disability and independent of any SSI receipt. Disability status does not remove an adult from the MAGI adult group, and MAGI-based groups take no resource test, so the $19,828 bank balance is not part of the determination. Head's MAGI of $940 is 0.06 x FPL, clearing the 138% FPL limit by a wide margin."
-us,scenario_080,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model stated the two premises that decide the case - Pennsylvania expanded Medicaid, and the financial assistance and investment income fall below the MAGI thresholds - then submitted 0, contradicting its own reasoning. Applying the threshold it had already named yields eligibility: $940 of MAGI is 0.06 x FPL against the 138% FPL adult expansion limit for a 41-year-old. It deferred the conclusion to 'PolicyEngine evaluates it' instead of completing the comparison it had set up."
-us,scenario_080,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for head_medicaid_eligible, so the submission carried no substantive determination to evaluate. The correct derivation is a 41-year-old in expansion-state Pennsylvania with MAGI of $940, or 0.06 x FPL, inside the 138% FPL ACA adult expansion limit, giving eligible."
-us,scenario_080,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of listed earnings and immigration facts as disqualifying, inverting the ACA adult expansion test, which is satisfied by low income rather than defeated by it: MAGI of $940 is 0.06 x FPL against the 138% FPL limit, the strongest possible qualifier. The prompt's stated default for unlisted status facts makes the head a qualified resident meeting the 5-year requirement, which is exactly the immigration finding the engine applied, and age 41 satisfies the 19-64 adult requirement."
-us,scenario_080,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted unnamed 'additional household financial and categorical criteria' beyond the four elements the ACA adult expansion group actually imposes - age 19-64, state residency, immigration status, and MAGI at or below 138% FPL - and read the near-total absence of income as evidence against eligibility rather than for it. A 41-year-old Pennsylvania resident with $940 of MAGI sits at 0.06 x FPL and meets every element of that group, which is the ADULT category the engine used."
-us,scenario_080,head_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the SSI-linked $2,000 resource limit from the aged/blind/disabled pathway, but the governing category here is the MAGI-based ACA adult expansion group, and 42 CFR 435.603 bars any asset or resource test for MAGI-based eligibility groups. The $19,828 bank balance and $3,570 vehicle value therefore never enter the determination; the only financial test is MAGI of $940, or 0.06 x FPL, against the 138% FPL limit."
+us,scenario_080,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model looked only at disability-based pathways, saying the head needed SSI or a disabled-adult category, and cited the $19,828 bank balance and 'disabled adult thresholds'. It never applied Pennsylvania's ACA expansion adult group, which has no asset test and covers anyone aged 19-64 with MAGI at or below 138% FPL. The head's MAGI is about $940, or 0.06x FPL, because the $3,600 of financial assistance is excluded from MAGI."
+us,scenario_080,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said the head's income was not low enough and that no qualifying category applied. In fact the head's MAGI is only about $940 of capital gains and dividends, 0.06x FPL, which is far below Pennsylvania's 138% FPL limit for the ACA expansion adult group. A 41-year-old in that group is eligible."
+us,scenario_080,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model correctly said Pennsylvania expanded Medicaid and that the head's income is below the MAGI thresholds, then answered 0 anyway. Under the expansion adult rule, a 41-year-old with MAGI of about $940 (0.06x FPL), which is under the 138% FPL limit, is eligible, so the model's own reasoning supports 1."
+us,scenario_080,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value or explanation for head_medicaid_eligible, so there was no answer to score. The correct answer is eligible, through Pennsylvania's ACA expansion adult group at MAGI of 0.06x FPL."
+us,scenario_080,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the lack of listed wages and immigration details as grounds for denial. Near-zero income is exactly what qualifies a 41-year-old for Pennsylvania's ACA expansion adult group: MAGI is about $940, or 0.06x FPL, against a 138% FPL limit. An unlisted immigration status does not block eligibility here."
+us,scenario_080,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said Medicaid needed more income or categorical criteria and answered 0 because no wages were listed and no other eligibility flags were set. Pennsylvania's ACA expansion adult group needs only an age of 19-64 and MAGI at or below 138% FPL, and the head's MAGI of about $940 (0.06x FPL) meets that test."
+us,scenario_080,head_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the $2,000 resource limit for disability-based (SSI-related) Medicaid and denied eligibility because of the $19,828 bank balance. MAGI-based coverage under Pennsylvania's ACA expansion adult group has no asset test, and the head's MAGI of about $940 (0.06x FPL) is well under the 138% FPL limit."
us,scenario_080,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model converted the generic disability flag directly into Medicare eligibility and assumed an SSDI-based pathway that was not provided. At age 41, disability alone does not establish Medicare entitlement without qualifying SSDI receipt and its waiting period, ALS, or end-stage renal disease."
us,scenario_080,head_medicare_eligible,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model misread PolicyEngine's rule as making every disabled person under 65 Medicare eligible. The disability input does not establish SSDI-based Medicare entitlement, ALS, or end-stage renal disease."
us,scenario_080,head_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated disability as qualifying for Medicare regardless of age. A 41-year-old needs a specific under-65 entitlement pathway, none of which is present in the household facts."
@@ -6490,52 +6660,53 @@ us,scenario_080,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F
us,scenario_080,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_080,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_080,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_080,snap,claude-fable-5,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE, the energy-subsidy SUA and the uncapped disabled shelter deduction to reach zero net income. It then paid a made-up $293/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also counted the $924 capital gains, which are not SNAP income, though that did not change the zero net income."
-us,scenario_080,snap,claude-fable-5.1,reference_engine_defect,categorical_eligibility,False,"Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum. But it reached zero net income the wrong way: it excluded the $300/month financial assistance, which is countable, and counted capital gains, instead of using the $857 heating SUA triggered by the energy subsidy."
-us,scenario_080,snap,claude-haiku-4.5,reference_engine_defect,categorical_eligibility,False,"It denied SNAP under a $2,750 resource limit and counted the $3,600 financial assistance as a resource. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test. The frozen reference reaches zero net income after the $857 SUA and pays the maximum allotment, $3,596.04, using the engine's projected FY2027 $304.68 for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 all year, which on that path gives $298 × 12 = $3,576."
-us,scenario_080,snap,claude-opus-4.7,reference_engine_defect,categorical_eligibility,False,"It paid a $293/month maximum for all 12 months instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also excluded the $300/month financial assistance, which is countable SNAP income, and counted capital gains, so it reached zero net income through the standard deduction alone rather than the energy-subsidy SUA."
-us,scenario_080,snap,claude-opus-4.8,reference_engine_defect,categorical_eligibility,False,"It applied the roughly $4,500 elderly/disabled asset limit to the $19,828 bank balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash removes the asset test for this low-income household."
-us,scenario_080,snap,claude-opus-5,reference_engine_defect,categorical_eligibility,False,"Its own reasoning says there is essentially no countable income, yet it submitted about $218.50/month instead of the maximum allotment. It never applied the $857 heating SUA from the energy subsidy that zeroes net income, and never used the FY2026 $298 maximum, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,claude-opus-5.5,reference_engine_defect,categorical_eligibility,False,"It paid $298 for all 12 months and knowingly left out any October 2026 cost-of-living update, which is what the release's SNAP convention (c_snap_hold_fy2026) does: October–December 2026 hold the FY2026 schedule because USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze. The frozen reference instead uses the engine's projected FY2027 maximum of $304.68 from October, giving 298×9 + 304.68×3 = $3,596.04. Its route to zero net income is wrong, though: $301.33 of countable income less the $209 standard deduction leaves $92.33, which the frozen reference zeroes only with the $857 heating SUA."
-us,scenario_080,snap,claude-sonnet-4.6,reference_engine_defect,categorical_eligibility,False,"It correctly applied BBCE, the energy-subsidy SUA and the uncapped disabled shelter deduction to reach zero net income. It then used the FY2025 $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also wrongly counted the $924 capital gains as SNAP income."
-us,scenario_080,snap,claude-sonnet-5,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 in liquid assets exceeds the $4,250 disabled-household limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test entirely."
-us,scenario_080,snap,claude-sonnet-5.5,reference_engine_defect,categorical_eligibility,False,"It said there were no shelter costs, missing that the $2,000 energy subsidy triggers Pennsylvania's $857 heating SUA through heat-and-eat. With the uncapped disabled excess shelter deduction, that SUA zeroes net income. It also counted the $924 capital gains, which pushed its net income up to $169."
-us,scenario_080,snap,deepseek-v4-flash-0731,reference_engine_defect,categorical_eligibility,False,"It stopped at the standard deduction and never applied the $857 heating SUA from the energy subsidy, which zeroes net income through the uncapped disabled shelter deduction. It also counted the $924 capital gains and used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December."
-us,scenario_080,snap,deepseek-v4-pro,reference_engine_defect,categorical_eligibility,False,"It excluded the energy subsidy from income but missed that the subsidy triggers the $857 heating SUA under heat-and-eat, which zeroes net income. It also counted capital gains and used a stale $292 maximum and $204 standard deduction."
-us,scenario_080,snap,deepseek-v4-pro-0813,reference_engine_defect,categorical_eligibility,False,"It applied no shelter deduction and missed the $857 heating SUA that the $2,000 energy subsidy triggers, which with the uncapped disabled excess shelter deduction reduces net income to $0. It also counted the $924 capital gains, and it used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December."
-us,scenario_080,snap,deepseek-v4.1-flash,reference_engine_defect,categorical_eligibility,False,"It applied the disabled-household asset limit to the $19,828 balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test."
-us,scenario_080,snap,gemini-3-flash-preview,reference_engine_defect,categorical_eligibility,False,"It correctly used the energy-subsidy SUA to reach zero net income, then paid the outdated $291/month one-person maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gemini-3.1-flash-lite-preview,reference_engine_defect,categorical_eligibility,False,"It zeroed SNAP because of the household's assets, missing that Pennsylvania's broad-based categorical eligibility waives the asset test. The household has $301.33/month of countable income, and with the $857 SUA it gets the maximum allotment."
-us,scenario_080,snap,gemini-3.1-pro-preview,reference_engine_defect,categorical_eligibility,False,"It correctly used the SUA from the energy subsidy to zero net income, then used the FY2025 $292/month maximum for all 12 months. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gemini-3.5-flash,reference_engine_defect,categorical_eligibility,False,"It reached zero net income through BBCE and the energy-subsidy SUA, but used a stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also used an understated $198 standard deduction and a $600 SUA instead of $209 and $857, and it counted capital gains as income."
-us,scenario_080,snap,gemini-3.5-flash-lite,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because of the household's assets and financial assistance. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test, and that $301.33/month of income less the standard deduction and $857 SUA leaves zero net income and the maximum allotment."
-us,scenario_080,snap,gemini-3.6-flash,reference_engine_defect,categorical_eligibility,False,"It correctly applied BBCE and the heating SUA from the energy subsidy to zero net income, then paid the FY2025 $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gemini-3.7-flash,reference_engine_defect,categorical_eligibility,False,"It paid the FY2026 $298 maximum for all 12 months, which matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference instead uses the engine's projected FY2027 increase to $304.68 for October–December 2026."
-us,scenario_080,snap,gemini-3.8-flash,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the energy-assistance heating SUA to zero net income, then paid the stale $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there was no answer to score against the $3,596.04 reference."
-us,scenario_080,snap,glm-5.3,reference_engine_defect,categorical_eligibility,False,"It noted the energy subsidy but used it only for categorical eligibility. It never applied the $857 heating SUA the subsidy triggers, which with the uncapped disabled shelter deduction zeroes net income. Instead it subtracted 30% of about $170 from a $290 maximum, after also counting the $924 capital gains."
-us,scenario_080,snap,gpt-5.4-mini,reference_engine_defect,categorical_eligibility,False,"It wrongly claimed the income and household details were insufficient and defaulted to $0. The facts fully define a one-person unit with $300/month financial assistance and $1.33 dividends, which under BBCE with the $857 SUA gets the maximum allotment: $3,596.04 in the frozen reference, which uses the engine's projected FY2027 $304.68 for October–December, or $298 × 12 = $3,576 under the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 all year."
-us,scenario_080,snap,gpt-5.4-nano,reference_engine_defect,categorical_eligibility,False,"It treated the absence of earned income as grounds for zero SNAP. Unearned income does not bar SNAP: a one-person unit with $301.33/month of unearned income qualifies under Pennsylvania BBCE, and the $857 SUA zeroes net income, giving the maximum allotment."
-us,scenario_080,snap,gpt-5.5,reference_engine_defect,categorical_eligibility,False,"It applied no shelter deduction and computed $298 minus 30% of $96, about $269/month. The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 × 12 = $3,240, $12 above the model because it used $96 of net income rather than $92. Its $298 maximum for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gpt-5.6-luna,reference_engine_defect,categorical_eligibility,False,"It computed $3,576 minus 30% of ($4,540 − $2,508) with no shelter deduction, so it never applied the $857 heating SUA from the energy subsidy, which zeroes net income. It also counted the $924 capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gpt-5.6-sol,reference_engine_defect,categorical_eligibility,False,"Its $270/month equals $298 minus 30% of the $92.33 left after the standard deduction, which is the result with no shelter deduction. Its $3,240 equals the exclusion's corrected value, which withholds the heat-and-eat SUA from a household without an elderly or disabled member; the model reached it by applying no SUA although it called the household disabled, a reading under which the record says the energy subsidy would still confer the SUA. Holding $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026). The frozen reference instead applies the $857 heating SUA the energy subsidy triggers, which zeroes net income, and uses the engine's projected FY2027 $304.68 for October–December."
-us,scenario_080,snap,gpt-5.6-terra,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 bank balance exceeds the disabled-household resource limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test."
-us,scenario_080,snap,gpt-6-astra,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the heating SUA to zero net income, then paid $298 for all 12 months. That matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December 2026; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gpt-6-luna,reference_engine_defect,categorical_eligibility,False,"It applied only the standard deduction and paid $269/month. The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 a month ($3,240), $1 above the model's $269. Keeping one maximum for all 12 months is consistent with the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum."
-us,scenario_080,snap,gpt-6-sol,reference_engine_defect,categorical_eligibility,False,"It counted the $940 of investment income, including $924 of capital gains that are not SNAP income, and applied only the $209 standard deduction. It never applied the $857 heating SUA from the energy subsidy, which zeroes net income and yields the maximum allotment."
-us,scenario_080,snap,gpt-6.1-sol,reference_engine_defect,categorical_eligibility,False,"It subtracted 30% of $2,032 net income from $3,576 with no shelter deduction, so it never applied the $857 heating SUA from the energy subsidy, which zeroes net income. It also counted capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum."
-us,scenario_080,snap,grok-4.3,reference_engine_defect,categorical_eligibility,False,"It wrongly treated the lack of shelter and dependent expenses as making SNAP zero, when missing expenses only reduce deductions. It also missed that the $2,000 energy subsidy triggers the $857 heating SUA, which zeroes net income and gives the maximum allotment."
-us,scenario_080,snap,grok-4.5,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the energy-subsidy SUA with the uncapped disabled shelter deduction, then paid the stale $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,grok-4.6,reference_engine_defect,categorical_eligibility,False,"It correctly reached zero net income through BBCE and the heating SUA, then multiplied the FY2025 $292 maximum by 12. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum."
-us,scenario_080,snap,grok-4.7,reference_engine_defect,categorical_eligibility,False,"Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum. But it counted only the $940 of investment income, wrongly excluding the $300/month financial assistance and wrongly including capital gains, instead of zeroing net income through the $857 SUA."
-us,scenario_080,snap,grok-build-0.1,reference_engine_defect,categorical_eligibility,False,"It denied SNAP under the $4,250 disabled-household asset limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test."
-us,scenario_080,snap,inkling,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the bank balance plus vehicle value exceed the disabled-household resource limit. It missed that Pennsylvania's broad-based categorical eligibility removes the resource test, and it wrongly treated school-age children as relevant to SNAP."
-us,scenario_080,snap,kimi-k2.6,reference_engine_defect,categorical_eligibility,False,"It applied the federal disabled-household resource limit to the $19,828 balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test."
-us,scenario_080,snap,kimi-k3,reference_engine_defect,categorical_eligibility,False,"It paid the stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also reached zero net income by treating investment income as the only countable income, missing the $300/month financial assistance and the $857 heating SUA from the energy subsidy that actually zeroes net income."
-us,scenario_080,snap,minimax-m3,reference_engine_defect,categorical_eligibility,False,"It treated the $19,828 bank balance plus the $3,600 financial assistance as disqualifying resources. It missed that Pennsylvania's broad-based categorical eligibility waives the SNAP asset test, and that the assistance is income, not a resource."
-us,scenario_080,snap,ox-alpha,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 balance exceeds the roughly $4,500 disabled-household asset limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash removes the asset test."
-us,scenario_080,snap,qwen-3.7-max,reference_engine_defect,categorical_eligibility,False,"It applied a $2,750 asset limit, which is also the wrong limit for a disabled household, and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility waives the asset test entirely."
-us,scenario_080,snap,qwen3.8-max,reference_engine_defect,categorical_eligibility,False,It denied SNAP because resources exceed the limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test for this one-person unit with $301.33/month of income.
+us,scenario_080,snap,claude-fable-5,reference_engine_defect,categorical_eligibility,False,"Correctly used the utility allowance from the energy subsidy to bring net income to $0, but applied a $293/month one-person maximum allotment for all 12 months. The correct maximum is $298/month for January–September and $304.68/month for October–December."
+us,scenario_080,snap,claude-fable-5.1,reference_engine_defect,categorical_eligibility,False,"Used $298/month for all 12 months and missed that October–December 2026 fall in FY2027, when the one-person maximum rises to $304.68. That leaves out $20.04. It also dropped the $300/month financial assistance from countable income; net income still comes out to $0 only because of the heat-and-eat utility allowance."
+us,scenario_080,snap,claude-haiku-4.5,reference_engine_defect,categorical_eligibility,False,"Applied a $2,750 SNAP resource limit and treated the $3,600 financial assistance as a resource. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test, so the household gets the full $3,596.04 maximum."
+us,scenario_080,snap,claude-haiku-5.5,reference_engine_defect,categorical_eligibility,False,"Denied SNAP because the $19,828 in liquid assets exceeded the resource limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test entirely."
+us,scenario_080,snap,claude-opus-4.7,reference_engine_defect,categorical_eligibility,False,"Reached $0 net income (by excluding the $300/month financial assistance instead of using the utility allowance) but used a $293/month maximum allotment ×12. The FY2026 one-person maximum is $298, rising to $304.68 from October."
+us,scenario_080,snap,claude-opus-4.8,reference_engine_defect,categorical_eligibility,False,"Applied the roughly $4,500 resource limit for households with an elderly or disabled member and found the household ineligible. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test."
+us,scenario_080,snap,claude-opus-5,reference_engine_defect,categorical_eligibility,False,"Its $218.50/month implies subtracting about 30% of a positive net income from the maximum, so it never applied the $857 heat-and-eat utility allowance from the $2,000 energy subsidy. That allowance plus the standard deduction brings net income to $0, which yields the full $298/$304.68 monthly maximum."
+us,scenario_080,snap,claude-opus-5.5,reference_engine_defect,categorical_eligibility,False,"Explicitly applied the FY2026 $298 maximum to all 12 months and ignored the October 2026 cost-of-living update. That update raises the one-person maximum to $304.68 for October–December, so the annual total is $3,596.04, not $3,576."
+us,scenario_080,snap,claude-sonnet-4.6,reference_engine_defect,categorical_eligibility,False,Correctly used the utility allowance to bring net income to $0 but used the FY2025 one-person maximum of $292/month ×12. The correct figures are $298/month for January–September and $304.68 for October–December.
+us,scenario_080,snap,claude-sonnet-5,reference_engine_defect,categorical_eligibility,False,"Failed the household on a $4,250 disabled-household asset limit. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program waives the asset test, so the $19,828 bank balance does not matter."
+us,scenario_080,snap,claude-sonnet-5.5,reference_engine_defect,categorical_eligibility,False,"Stated there were no shelter costs and never applied Pennsylvania's $857 utility allowance, which the $2,000 energy subsidy confers through heat-and-eat. It also counted the $924 in capital gains as income, leaving $169 of net income instead of $0, then subtracted 30% of it."
+us,scenario_080,snap,deepseek-v4-flash-0731,reference_engine_defect,categorical_eligibility,False,"Counted the $924 in capital gains and applied only the standard deduction, leaving $170 of net income. It skipped the $857 heat-and-eat utility allowance triggered by the energy subsidy, which brings net income to $0 and yields the full maximum allotment."
+us,scenario_080,snap,deepseek-v4-pro,reference_engine_defect,categorical_eligibility,False,"Correctly excluded the energy subsidy from income but missed that it triggers Pennsylvania's $857 utility allowance through heat-and-eat, so it left $174 of net income. It also used the outdated $292 maximum instead of $298/$304.68."
+us,scenario_080,snap,deepseek-v4-pro-0813,reference_engine_defect,categorical_eligibility,False,"Assumed there was no shelter deduction and left $168 of net income. It missed that the $2,000 energy subsidy entitles the household to the $857 heat-and-eat utility allowance, which brings net income to $0. It also counted capital gains as SNAP income."
+us,scenario_080,snap,deepseek-v4.1-flash,reference_engine_defect,categorical_eligibility,False,Denied SNAP on the asset limit for disabled-member households. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test.
+us,scenario_080,snap,gemini-3-flash-preview,reference_engine_defect,categorical_eligibility,False,Correctly used the utility allowance to bring net income to $0 but used a $291/month maximum ×12. The one-person maximum is $298 for January–September and $304.68 for October–December.
+us,scenario_080,snap,gemini-3.1-flash-lite-preview,reference_engine_defect,categorical_eligibility,False,Cited the household's assets and lack of income as grounds for $0. It missed that Pennsylvania's categorical eligibility waives the asset test and that the $0 net income it would have found yields the maximum benefit.
+us,scenario_080,snap,gemini-3.1-pro-preview,reference_engine_defect,categorical_eligibility,False,Correctly reached $0 net income through the utility allowance but used the FY2025 $292/month maximum ×12. The correct maximums are $298 for January–September and $304.68 for October–December.
+us,scenario_080,snap,gemini-3.5-flash,reference_engine_defect,categorical_eligibility,False,"Correctly reached $0 net income through the energy-assistance utility allowance but applied the FY2025 $292/month maximum for all 12 months. The FY2026 maximum is $298, and $304.68 applies from October."
+us,scenario_080,snap,gemini-3.5-flash-lite,reference_engine_defect,categorical_eligibility,False,"Zeroed the benefit because of the household's assets. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test and that the household's net income is $0, which yields the maximum allotment."
+us,scenario_080,snap,gemini-3.6-flash,reference_engine_defect,categorical_eligibility,False,Correctly applied broad-based categorical eligibility and the heating utility allowance but used a $292/month maximum ×12. The correct figures are $298/month for January–September and $304.68 for October–December.
+us,scenario_080,snap,gemini-3.7-flash,reference_engine_defect,categorical_eligibility,False,"Applied the $298 FY2026 maximum to all 12 months. It missed the FY2027 increase to $304.68/month for October–December 2026, which adds $20.04 to the annual total."
+us,scenario_080,snap,gemini-3.8-flash,reference_engine_defect,categorical_eligibility,False,Correctly used broad-based categorical eligibility and the heating utility allowance to reach $0 net income but used the outdated $292/month maximum ×12. The correct maximums are $298 and $304.68.
+us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score."
+us,scenario_080,snap,glm-5.3,reference_engine_defect,categorical_eligibility,False,"Recognized the energy subsidy as LIHEAP-type assistance but used it only for categorical eligibility, not to trigger the $857 utility allowance. It therefore left about $170 of net income and also used an outdated $290 maximum."
+us,scenario_080,snap,gpt-5.4-mini,reference_engine_defect,categorical_eligibility,False,"Claimed the facts were insufficient and returned $0. In fact the one-person household with $300/month assistance qualifies through Pennsylvania's broad-based categorical eligibility and the heat-and-eat utility allowance, which brings net income to $0 and pays the maximum allotment."
+us,scenario_080,snap,gpt-5.4-nano,reference_engine_defect,categorical_eligibility,False,"Set SNAP to $0 for lack of earned income and expense details. SNAP needs no earned income: the household's $301 of monthly income, Pennsylvania's broad-based categorical eligibility, and the $857 utility allowance from the energy subsidy produce the full maximum benefit."
+us,scenario_080,snap,gpt-5.5,reference_engine_defect,categorical_eligibility,False,"Stated there was no shelter deduction and left $96 of net income. It missed that the $2,000 energy subsidy triggers the $857 heat-and-eat utility allowance, which brings net income to $0. It also did not apply the $304.68 October–December maximum."
+us,scenario_080,snap,gpt-5.6-luna,reference_engine_defect,categorical_eligibility,False,"Counted the $924 in capital gains as SNAP income and applied no shelter deduction. It missed the $857 heat-and-eat utility allowance from the energy subsidy, which brings net income to $0 and yields the full maximum allotment."
+us,scenario_080,snap,gpt-5.6-sol,reference_engine_defect,categorical_eligibility,False,"Reduced income only by the standard deduction and subtracted 30% of the remainder, reaching $270/month. It never applied the heat-and-eat utility allowance from the $2,000 energy subsidy, which brings net income to $0."
+us,scenario_080,snap,gpt-5.6-terra,reference_engine_defect,categorical_eligibility,False,"Applied the SNAP resource limit to the $19,828 bank balance. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test."
+us,scenario_080,snap,gpt-6-astra,reference_engine_defect,categorical_eligibility,False,Correctly used categorical eligibility and the heating utility allowance to reach the maximum benefit but paid $298 for all 12 months. It missed the October 2026 increase to $304.68/month for the FY2027 months.
+us,scenario_080,snap,gpt-6-luna,reference_engine_defect,categorical_eligibility,False,"Applied only the standard deduction to the $300/month assistance and subtracted 30% of the remainder. It missed that the energy subsidy triggers the $857 heat-and-eat utility allowance, which brings net income to $0 and pays the full maximum."
+us,scenario_080,snap,gpt-6-sol,reference_engine_defect,categorical_eligibility,False,Counted $940 of investment income (including the $924 in capital gains) and applied only the $209 standard deduction. It omitted the $857 heat-and-eat utility allowance that brings net income to $0.
+us,scenario_080,snap,gpt-6.1-sol,reference_engine_defect,categorical_eligibility,False,"Subtracted 30% of $2,032 in annual net income from $3,576. That income figure counts capital gains and skips the $857 heat-and-eat utility allowance, which brings net income to $0. It also missed the $304.68 maximum for October–December."
+us,scenario_080,snap,grok-4.3,reference_engine_defect,categorical_eligibility,False,"Treated the absence of shelter expenses as producing a $0 benefit, which reverses how the formula works: lower net income raises the benefit. It also missed that the energy subsidy confers the $857 utility allowance, so net income is $0 and the household receives the maximum allotment."
+us,scenario_080,snap,grok-4.5,reference_engine_defect,categorical_eligibility,False,Correctly applied broad-based categorical eligibility and the utility allowance to reach $0 net income but used the $292/month FY2025 maximum ×12. The correct maximums are $298 for January–September and $304.68 for October–December.
+us,scenario_080,snap,grok-4.6,reference_engine_defect,categorical_eligibility,False,Correctly reached $0 net income through the heating utility allowance but used a $292/month maximum ×12. The correct one-person allotments are $298 for January–September and $304.68 for October–December.
+us,scenario_080,snap,grok-4.7,reference_engine_defect,categorical_eligibility,False,"Used $298 for all 12 months, missing the FY2027 increase to $304.68/month for October–December 2026. It also excluded the $300/month assistance from income, though net income still comes out to $0 because of the utility allowance."
+us,scenario_080,snap,grok-build-0.1,reference_engine_defect,categorical_eligibility,False,"Applied the $4,250 disabled-household asset limit and found the household ineligible. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program waives the asset test."
+us,scenario_080,snap,inkling,reference_engine_defect,categorical_eligibility,False,Denied SNAP on the resource limit and wrongly tied eligibility to school-age children. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test for this one-person household.
+us,scenario_080,snap,kimi-k2.6,reference_engine_defect,categorical_eligibility,False,"Applied the federal resource limit for disabled-member households. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test, so the household gets the maximum allotment."
+us,scenario_080,snap,kimi-k3,reference_engine_defect,categorical_eligibility,False,"Reached $0 net income (by excluding the financial assistance) and correctly waived the asset test, but used a $292/month maximum ×12. The correct allotments are $298 for January–September and $304.68 for October–December."
+us,scenario_080,snap,minimax-m3,reference_engine_defect,categorical_eligibility,False,"Treated the $19,828 in assets (plus the assistance) as disqualifying. It missed that Pennsylvania's broad-based categorical eligibility waives the asset test and that net income is $0, which pays the maximum allotment."
+us,scenario_080,snap,ox-alpha,reference_engine_defect,categorical_eligibility,False,"Applied the roughly $4,500 disabled-household asset limit. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test entirely."
+us,scenario_080,snap,qwen-3.7-max,reference_engine_defect,categorical_eligibility,False,"Applied the $2,750 general asset limit, ignoring both the higher limit for disabled households and, decisively, Pennsylvania's broad-based categorical eligibility, which waives the asset test."
+us,scenario_080,snap,qwen3.8-max,reference_engine_defect,categorical_eligibility,False,"Denied SNAP because resources exceeded the limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test, and that a lack of earned income does not prevent SNAP eligibility."
us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, violating the required structured-output contract."
us,scenario_080,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the prompt's general disability fact as establishing SSI disability, despite the applicable engine input is_ssi_aged_blind_disabled=False. Because the 41-year-old is neither SSI-aged, blind, nor SSI-disabled, the federal base-rate and unearned-income calculation does not apply and SSI is $0."
us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived 100% Pennsylvania Tax Forgiveness and explicitly calculated that it reduces the $28.86 preliminary tax to $0, but then submitted the unreduced $28.86. It failed to carry its own nonrefundable-credit calculation into the output."
@@ -6549,134 +6720,137 @@ us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_erro
us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable output for state_refundable_credits, violating the required submission contract."
us,scenario_080,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model misclassified Pennsylvania Schedule SP tax forgiveness as a refundable credit that pays out the $28.86 liability. Schedule SP is nonrefundable and only reduces state income tax, so it contributes $0 to state_refundable_credits."
us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Massachusetts income tax withholding as 0 and counted only the $5,643 real estate tax as SALT, so it itemized $22,543 for taxable income of about $151,550. The prompt lists no state income tax paid; the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld (5% of federal AGI above the $4,400 exemption), for $31,027.41 itemized and $143,059.47 taxable. With no state income tax paid, the engine's SALT takes $853 of general sales tax and the output is $28,763.81. It then used a wrong $17,754.75 base for the 24% bracket (the 2026 figure is $17,966) and added an unexplained ~$626 on top, giving $29,385."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted about $8,230 of Massachusetts income tax, the same figure as its Massachusetts income tax answer and close to the engine's own $8,238.41 liability; the reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld. It also added the $6 REIT/PTP income on top of dividends, making AGI $174,094 against the engine's $174,088.12. With the engine's liability in SALT the output is $26,991.31; its taxable income of about $143,320 and $26,995 compare with that reading's $143,305.47 and $26,991.31, and with the reference's $143,059.47 and $26,932.27."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It took a 2024-level $14,600 standard deduction and claimed the $16,900 of mortgage interest was below that threshold. In fact mortgage interest alone exceeds even that, and the engine itemizes $31,027.41 in the reference, whose SALT adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, and $23,396 with no state income tax paid, both above the $16,100 2026 standard deduction. It also subtracted the $1,782 of unreimbursed employee business expenses, which are nondeductible, used 2024 brackets, and then submitted $29,825, which differs from its own $30,956.92 result."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000, writing 'real estate $5,643 + state income tax well over cap', but the 2026 cap is $40,400, so SALT is deductible in full: $14,127.41 in the reference (real estate tax plus the engine's $8,484.41 estimate of Massachusetts income tax withheld) or $13,881.41 with the engine's own $8,238.41 Massachusetts tax. Against the reference the cap cut $4,127.41 from itemized deductions and pushed taxable income to $147,187. It then dropped its own $27,887 bracket result and submitted an unsupported $31,369."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired $10,000 SALT cap instead of the 2026 $40,400 cap, cutting $4,127.41 from the reference's $14,127.41 of SALT (real estate tax plus the engine's $8,484.41 estimate of Massachusetts income tax withheld; $13,881.41 with the engine's own Massachusetts tax). It also used 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 ones (12,400/50,400/105,700). Its own computation reached $28,172, but it submitted an unexplained $31,379."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($22,543), with no state income tax in SALT, which the prompt does not list. The reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, for taxable income of $143,059.47 against its $151,544; with no state income tax paid, the engine's SALT takes $853 of general sales tax and the output is $28,763.81. Its correct 2026 bracket math gave $28,968, which it then inflated without basis to $31,650."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,238.70 of Massachusetts income tax, the same figure as its Massachusetts income tax answer (5% of wages after the FICA, personal exemption and rent deductions, plus 5% of its interest and dividends), 29 cents above the engine's own $8,238.41 liability. The reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld, 5% of $169,688.12. It also counted dividends as $116 by adding the $6 REIT income. Its taxable income of $143,311.10 and $26,992.66 compare with $143,305.47 and $26,991.31 with the engine's liability in SALT; with the corrected liability ($8,232.90; the household's Massachusetts output is excluded for an engine defect) the output is $26,992.64, within 2 cents of its answer."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 real estate tax toward SALT, with no state income tax, which the prompt does not list, so it itemized $22,543 and reached taxable income of $151,551. The reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, for $31,027.41 itemized; with no state income tax paid, the engine's SALT takes $853 of general sales tax and the output is $28,763.81. It then applied 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 ones (12,400/50,400/105,700), giving $29,219."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 $40,400 cap, understating itemized deductions by $4,127.41 against the reference, whose SALT adds the engine's $8,484.41 estimate of Massachusetts income tax withheld ($3,881.41 against the engine's own Massachusetts tax). On its own $147,194 of taxable income, the 2026 single brackets give about $27,925. Its $33,700 does not follow from any bracket computation it showed."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted about $8,284 of Massachusetts income tax, the same figure as its Massachusetts income tax answer and $45.59 above the engine's own $8,238.41 liability; the reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld. It also added the $6 REIT income to AGI. Its taxable income of about $143,266 and $26,982 compare with $143,305.47 and $26,991.31 with the engine's liability in SALT, and with $143,059.47 and $26,932.27 in the reference."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap even though OBBBA raises the 2026 cap to $40,400. That cut $4,127.41 from the reference's $14,127.41 of SALT (real estate tax plus the engine's $8,484.41 estimate of Massachusetts income tax withheld) and left taxable income at $147,194 instead of the reference's $143,059.47. It also overshot the correct 2026 bracket result on that income ($27,925) by about $54."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a TCJA sunset: pre-TCJA 10/15/25/28% brackets and a $5,400 personal exemption. OBBBA made the TCJA rate schedule and the $0 exemption permanent for 2026. It also subtracted the $7,389 ESI premium from the $175,002 wages, which the engine counts in full, and put no state income tax in SALT, which the prompt does not list (the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld)."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law, with a $5,312 personal exemption and the 10/15/25/28% schedule, even though OBBBA made the TCJA brackets and the zero exemption permanent for 2026. It also put SALT at $12,754.55 of property and state tax, which implies $7,111.55 of state income tax, below both the engine's own $8,238.41 Massachusetts tax and the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses ($14,127.41 of SALT). Its submitted $27,594.14 contradicts the $29,794.41 in its own explanation."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its structure matches the engine's, but it deducted $7,539.70 of Massachusetts income tax, the same figure as its Massachusetts income tax answer, which is $698.71 below the engine's own $8,238.41 liability and $944.71 below the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses. It also added the $6 REIT income to AGI. Its taxable income of $144,010.10 and $27,160.42 compare with $143,305.47 and $26,991.31 with the engine's liability in SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired for 2026, so it subtracted a $5,050 personal exemption and taxed the income under 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 personal exemption permanent. Its itemized deductions ($30,823) were close; the error is the rate schedule and the exemption."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing $16,900 of mortgage interest plus SALT, which the reference puts at $14,127.41 with the engine's $8,484.41 estimate of Massachusetts income tax withheld ($31,027.41 in all; $23,396 with no state income tax paid), overstating taxable income by about $16,000 against the reference. Even on its own $159,094 of taxable income, the 2026 brackets give $30,781, not the $31,317 it submitted."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-expired brackets and a personal exemption, although OBBBA made the TCJA 10/12/22/24% schedule and the zero exemption permanent for 2026. It also reduced AGI to $166,705 by subtracting the $7,389 ESI premium from wages, while the engine counts the full $175,002 as employment income."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired, with a $5,300 personal exemption and pre-TCJA brackets, although OBBBA keeps the 10/12/22/24% schedule and the $0 exemption for 2026. It also subtracted the $7,389 ESI premium from the $175,002 wages and deducted only the real estate tax, with no state income tax in SALT (the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, which the prompt does not list)."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $169,008 of taxable income is only about $5,000 below AGI, so it left out most of the itemized mortgage interest and SALT: $31,027.41 in the reference, whose SALT adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, and $23,396 with no state income tax paid. Its $24,197 also does not follow from 2026 single brackets on $169,008, which give about $33,160."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $7,389 ESI premium from the $175,002 wages, which the engine counts in full. It also put no state income tax in SALT, itemizing only $22,543; the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, which the prompt does not list. These two choices offset each other into taxable income of $144,151, on which the 2026 brackets give $27,194; it then misapplied the brackets and reported $27,858."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The reference taxes $143,059.47 at 24% above $105,700, which yields $26,932.27; its itemized deductions of $31,027.41 include the engine's $8,484.41 estimate of Massachusetts income tax withheld. The model's $24,907 implies taxable income of about $134,620, roughly $8,440 of deductions beyond the reference's. That is consistent with subtracting the $7,389 ESI premium from wages and over-deducting on top of the reference's itemized deductions."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It put Massachusetts income tax at $8,238.40, the same figure as its Massachusetts income tax answer and within a cent of the engine's own $8,238.41 liability, but then capped total SALT at $10,000. The 2026 cap is $40,400, so SALT is deductible in full: $13,881.41 with that liability, for an output of $26,991.31, and $14,127.41 in the reference, which uses the engine's $8,484.41 estimate of Massachusetts income tax withheld. The cap put its taxable income at $147,187, and it then overshot the 2026 bracket tax on that income ($27,923) by $127."
-us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset law, subtracting a $5,150 personal exemption and applying 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 exemption permanent. Its uncapped SALT treatment was close in effect, since the actual $40,400 cap does not bind, but the wrong rate schedule raised tax to $29,110."
-us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It put Massachusetts income tax at $8,388.70, the same figure as its Massachusetts income tax answer, between the engine's own $8,238.41 liability and the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses, and it added the $6 REIT income to AGI, leaving taxable income at $143,161.10. On that figure its own bracket components sum to $26,956.66. It mis-added them and reported $26,916.66, $40 below its own components."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 $40,400 cap, cutting $4,127.41 from the reference's $14,127.41 of SALT (real estate tax plus the engine's $8,484.41 estimate of Massachusetts income tax withheld). With that cap, taxable income is about $147,187 and 2026 tax is about $27,923. Its $22,450 is roughly $5,500 below even that, so it also misapplied the single rate schedule."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,495 exceeds the $34,379 that the 2026 single brackets produce on the full $174,088 AGI with no deduction at all. So it neither subtracted its itemized deductions ($31,027.41 in the reference, $23,396 with no state income tax paid) nor applied the 10/12/22/24% schedule, whose 24% bracket runs to $201,775."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions (about $30,933) were close to the reference's $31,027.41, whose SALT adds the engine's $8,484.41 estimate of Massachusetts income tax withheld. But it added a 2026 personal exemption and taxed the result under the pre-TCJA rate schedule, while OBBBA permanently sets the exemption to $0 and keeps the 10/12/22/24% brackets. That pushed tax to $29,417 instead of $26,932."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced wages by the $7,389 employer-sponsored insurance premium, but the engine treats the full $175,002 as taxable employment income. That removes about $7,400 of taxable income and roughly $1,770–$2,000 of tax at 24%, producing $24,920 instead of $26,932.27."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemization, 2026 brackets and QBI treatment match the engine's. It gave no state tax figure in this answer; its Massachusetts income tax answer is $8,238.70, and its $26,992.59 is consistent with deducting about that in SALT, where the reference uses the engine's $8,484.41 estimate of Massachusetts income tax withheld. With the engine's own $8,238.41 liability in SALT the output is $26,991.31, and with the corrected liability ($8,232.90; the household's Massachusetts output is excluded for an engine defect) it is $26,992.64, within 5 cents of its answer."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its approach matches the engine's: itemize mortgage interest plus Massachusetts income and property tax, then apply the 2026 24% bracket. Its Massachusetts income tax answer is $8,338.70, $100.29 above the engine's own $8,238.41 liability and $145.71 below the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses. Its $26,966.21 is $33.94 above the reference and $25.10 below the $26,991.31 the engine gives with its liability in SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions implies $8,230.10 of Massachusetts income tax, the same figure as its Massachusetts income tax answer and $8.31 below the engine's own $8,238.41 liability. The reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld, 5% of $169,688.12. With the corrected liability ($8,232.90; the household's Massachusetts output is excluded for an engine defect) the output is $26,992.64, and its $26,993.29 is within $1 of that; the reference is $26,932.27."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its method matches the engine's. Its taxable income of about $143,260 is consistent with deducting $8,284, its Massachusetts income tax answer, which is $45.59 above the engine's own $8,238.41 liability; the reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld, for $143,059.47. Its $26,980 is $47.73 above the reference and $11.31 below the $26,991.31 the engine gives with its liability in SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,330.10 of Massachusetts income tax, the same figure as its Massachusetts income tax answer, $91.69 above the engine's own $8,238.41 liability and $154.31 below the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses. Its taxable income of $143,213.70 and $26,969.29 compare with $143,305.47 and $26,991.31 with the engine's liability in SALT, and with $143,059.47 and $26,932.27 in the reference."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions implies $8,230.10 of Massachusetts income tax, the same figure as its Massachusetts income tax answer and $8.31 below the engine's own $8,238.41 liability. The reference instead uses the engine's $8,484.41 estimate of Massachusetts income tax withheld, 5% of $169,688.12, for taxable income of $143,059.47; the model's works out to $143,313.70. With the corrected liability ($8,232.90; the household's Massachusetts output is excluded for an engine defect) the output is $26,992.64, and its $26,993.29 is within $1 of that."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 is roughly $8,440 above the reference's $143,059.47. That matches itemizing only mortgage interest and real estate tax, with no state income tax in SALT; the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, which the prompt does not list. With no state income tax paid, the engine's SALT takes $853 of general sales tax and the output is $28,763.81, against its $28,950."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset, subtracting a $5,447 personal exemption and applying restored 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 exemption permanent. Its itemized deductions ($30,939, with $8,396 of Massachusetts income tax) were close to the reference's $31,027.41, so the error is the wrong rate schedule and the phantom exemption."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a TCJA sunset, with a restored $5,300 personal exemption and 10/15/25/28% brackets, although OBBBA made the TCJA rates and the zero exemption permanent for 2026. Its $30,932 of itemized deductions, with $8,389 of Massachusetts income tax, was close to the reference's $31,027.41, but the pre-TCJA schedule pushed tax to $29,479."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law, with a $5,350 personal exemption and restored 10/15/25/28% brackets, although OBBBA keeps the 10/12/22/24% schedule and the $0 exemption for 2026. Its $30,827.70 of itemized deductions, with $8,284.70 of Massachusetts income tax, was close to the reference's $31,027.41, so the rate-schedule error drives its $29,389."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied reverted pre-TCJA brackets (10% to $12,491, 15%, 25%, 28%) instead of the permanent OBBBA 10/12/22/24% schedule. It also itemized only $22,543, with no state income tax in SALT, so taxable income was $151,551; the reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, which the prompt does not list, for $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted about $8,335 of Massachusetts income tax, the same figure as its Massachusetts income tax answer, between the engine's own $8,238.41 liability and the engine's $8,484.41 estimate of Massachusetts income tax withheld that the reference uses, giving taxable income of about $143,215. On that figure the 2026 brackets give $26,970, and it then misapplied the 24% bracket, adding about $86 more to reach $27,056."
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $16,900 of mortgage interest and $5,643 of real estate tax, with no state income tax in SALT, which the prompt does not list. Taxable income was $151,543.80, and its otherwise-correct 2026 bracket math gave $28,968.51. The reference adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, for $143,059.47; with no state income tax paid, the engine's SALT takes $853 of general sales tax and the output is $28,763.81."
-us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It reported $0 with no derivation. With $174,088.12 of AGI, the engine's tax is $26,932.27 in the reference (taxable income $143,059.47, with SALT that adds the engine's $8,484.41 estimate of Massachusetts income tax withheld) and more under the other readings of that unlisted tax, with no nonrefundable credit to offset it."
-us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,538 of Massachusetts income tax, the same figure as its Massachusetts income tax answer: 5% of wages above the $4,400 exemption plus 5% of interest and dividends, without netting the $1,080 short-term loss. The reference uses the engine's $8,484.41 estimate of Massachusetts income tax withheld, 5% of federal AGI above the exemption, which nets the loss; the engine's own Massachusetts liability is $8,238.41. The excess $53.59 over the estimate, partly offset by adding $6 of REIT income to AGI, left taxable income $47 below the reference at about $143,012, and its $26,921 is $11.27 below the reference and $70.31 below the $26,991.31 with the engine's liability in SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the $22,543 of itemized deductions was lower than a $15,000 standard deduction, which is false, and it put no state income tax in SALT. The reference itemizes $31,027.41, with SALT that adds the engine's $8,484.41 estimate of Massachusetts income tax withheld, and even with no state income tax paid the engine itemizes $23,396; both exceed the $16,100 2026 standard deduction. It then used non-2026 brackets and submitted $39,827.88, which contradicts its own $31,281.32 calculation."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It chose the standard deduction on the claim that its $22,543 of itemized deductions was lower, though $22,543 already exceeds the $16,100 2026 standard deduction. It put no state income tax in SALT; the reference itemizes $31,027.41 with SALT that adds the engine's $8,484.41 estimate of Massachusetts income tax withheld. That left taxable income at $160,402, and even there the 2026 brackets give $31,094, not the $30,041 it reported."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It set the Massachusetts income tax in SALT to zero and deducted only the $5,643 property tax. That dropped the $8,484.41 estimated MA withholding that fits under the $40,400 OBBBA cap and overstated taxable income at about $151,550. It then raised its own $28,759 bracket result to $29,385 with no stated reason."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,230 for MA income tax in SALT, which is the final MA liability, instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. Itemized deductions came out about $254 too low, taxable income about $260 too high ($143,320 vs $143,059.47), and the tax about $63 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $14,600 standard deduction (the 2024 figure) and wrongly said the $16,900 mortgage interest did not justify itemizing. It skipped the $31,027.41 of itemized deductions (mortgage interest + property tax + MA income tax under the $40,400 SALT cap). It also deducted the $1,782 of unreimbursed employee expenses above the line, used 2024 brackets, and submitted a number that does not match its own $30,956.92."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $33,000 is a round guess with no derivation. It is higher than even the $30,514.85 tax under the $16,100 standard deduction, so it never applied the $31,027.41 of itemized deductions (mortgage interest $16,900 + SALT $14,127.41) that bring taxable income down to $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, which cut $4,127.41 of deductible MA income and property tax. It then replaced its own $27,887 result with an unsupported $31,369."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible state and local tax. It also used 2025 bracket thresholds. Finally it swapped its own $28,172 for an unexplained $31,379."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax withholding estimate, so taxable income was overstated at about $151,544. It then replaced its own $28,968 bracket result with an unsupported $31,650."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,238.70 MA income tax liability in SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. It also counted the $6 REIT income twice in AGI ($116 of dividends, AGI $174,094). Together these made taxable income about $252 too high and the tax about $60 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax deduction allowed under the $40,400 cap. It also used 2025 bracket thresholds ($11,925/$48,475/$103,350) instead of the 2026 ones ($12,400/$50,400/$105,700)."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductions. It then gave $33,700 without a bracket calculation; the correct brackets applied to its own $147,194 taxable income give about $27,925."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated MA income tax at about $8,284 instead of PolicyEngine's withholding-based $8,484.41 (5% × (AGI − $4,400)). Its taxable income was about $200 too high ($143,266) and the tax about $50 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible MA income and property tax. It also counted the $6 REIT income twice in AGI and skipped the QBI deduction."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and applied a $5,400 personal exemption with 10/15/25/28% brackets, but OBBBA made the TCJA rates permanent with no personal exemption. It also subtracted the $7,389 ESI premiums from the stated gross wages and left MA income tax out of SALT."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,312 personal exemption and pre-TCJA brackets, instead of OBBBA's permanent 2026 brackets with no exemption. It also understated SALT at $12,754.55 against the correct $14,127.41. Its submitted $27,594.14 does not even match the $29,794.41 its own reasoning reached."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It put MA income tax in SALT at $7,539.70 instead of PolicyEngine's withholding estimate of $8,484.41 (5% × (AGI − $4,400)), understating itemized deductions by about $945. It also counted the $6 REIT income twice in AGI, which pushed taxable income to $144,010.10."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules, a $5,050 personal exemption and 10/15/25/28% brackets, but under OBBBA 2026 keeps the 10/12/22/24% schedule with no personal exemption. This produced a different taxable-income base and a different rate schedule."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction. It never itemized the $16,900 mortgage interest plus $14,127.41 SALT (property tax and MA income tax), which totals $31,027.41 and is far above the standard deduction."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used TCJA-expired brackets and a personal exemption for 2026 instead of OBBBA's permanent 10/12/22/24% schedule. It also cut wages by the $7,389 ESI premiums, giving an AGI of $166,705 instead of $174,088.12."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired and applied a $5,300 personal exemption with pre-TCJA brackets instead of OBBBA's 2026 schedule. It also subtracted $7,389 of ESI premiums from the stated gross wages and left MA income tax out of SALT."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its stated taxable income of $169,008 leaves out nearly all of the $31,027.41 itemized deductions. Its $24,197 also does not follow from the 2026 brackets applied to that income (about $33,160), so both the deduction step and the rate step are wrong."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $7,389 ESI premiums from the stated gross wages, which PolicyEngine treats as the taxable wage base. It also left the $8,484.41 MA income tax out of SALT and counted only the $5,643 property tax. Its $27,858 does not match the 2026 brackets applied to its own $144,151 taxable income."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $24,907 matches about $134,600 of taxable income under the 2026 brackets, roughly $8,400 below the correct $143,059.47. That means it over-deducted beyond the $31,027.41 of itemized deductions (mortgage interest + property tax + $8,484.41 MA withholding estimate) and the $1.24 QBI deduction."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible MA income and property tax. Taxable income came out at $147,187 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset and applied a $5,150 personal exemption with 10/15/25/28% brackets, but OBBBA made the 10/12/22/24% schedule permanent with no exemption. Its SALT figure of $14,348 was close to the correct $14,127.41; the error came from the law regime."
+us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,388.70 for MA income tax instead of the $8,484.41 withholding estimate and counted the $6 REIT income twice in AGI. It also made an arithmetic error: $17,966 + 24% × $37,461.10 is $26,956.66, not the $26,916.66 it submitted."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400. Even with that capped deduction, its $22,450 is about $5,500 below what the 2026 brackets give, so it misapplied the rate schedule as well."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,495 is higher than the $34,379 tax on the full $174,088 AGI with no deductions at all. So it applied none of the $31,027.41 itemized deductions and overstated the bracket tax as well."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a 2026 personal exemption that OBBBA permanently set to zero and applied the pre-TCJA rate schedule. That overstated the tax even though its itemized deductions of about $30,933 were close to the correct $31,027.41."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced wages by the $7,389 ESI premium, but the stated $175,002 gross wages is the federal wage base PolicyEngine taxes. This understated AGI and taxable income by several thousand dollars, so the tax came out about $2,000 too low."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted the Massachusetts income tax liability (about $8,238) as SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. Itemized deductions came out about $246 short, and the tax $60 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It understated the MA income tax in SALT by about $140 compared with PolicyEngine's $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income was about $140 too high and the tax $34 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions uses about $8,230 of MA income tax (the liability) instead of the $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income came out at $143,313.70 instead of $143,059.47, and the tax $61 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted about $200 less MA income tax than PolicyEngine's $8,484.41 withholding estimate. Its taxable income of about $143,260 is above the correct $143,059.47, so the tax came out $48 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,330.10 of MA income tax instead of the $8,484.41 withholding estimate, which is 5% of AGI above the $4,400 exemption. Taxable income came out at $143,213.70 instead of $143,059.47, overstating the tax by $37."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions uses about $8,230 of MA income tax (the liability) instead of PolicyEngine's $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income came out at $143,313.70 and the tax $61 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 means it deducted only mortgage interest and property tax and left out the $8,484.41 MA income tax deduction allowed under the $40,400 OBBBA SALT cap."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset in 2026 and applied a $5,447 personal exemption and 10/15/25/28% brackets, but OBBBA made the 10/12/22/24% schedule permanent with zero personal exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules, a $5,300 personal exemption and restored 10/15/25/28% brackets, instead of OBBBA's permanent 2026 schedule with no exemption. Its itemized total of $30,932 was nearly right, but the rate regime was wrong."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset, with a $5,350 personal exemption and 10/15/25/28% brackets, though OBBBA kept the 10/12/22/24% brackets and repealed the exemption permanently. The rate schedule pushed its tax $2,457 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA reverted brackets (10/15/25/28%) instead of OBBBA's permanent 2026 schedule. It also left the $8,484.41 MA income tax out of SALT, deducting only the $5,643 property tax."
+us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,335 of MA income tax instead of the $8,484.41 withholding estimate and counted the $6 REIT income twice in AGI. It also misapplied the brackets: its own $143,215 taxable income gives about $26,970 under the 2026 schedule, not $27,056."
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax withholding estimate that is deductible under the $40,400 OBBBA cap. Taxable income came out at $151,543.80 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It reported $0 tax with no derivation, even though taxable income of $143,059.47 at the 2026 single brackets produces $26,932.27 and no nonrefundable credits apply."
+us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated the MA income tax in SALT at $8,538 against PolicyEngine's $8,484.41, which is 5% × (AGI $174,088.12 − $4,400). It also counted the $6 REIT income twice in AGI. Taxable income came out at about $143,012 instead of $143,059.47, so the tax was $11 too low."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly said $22,543 of itemized deductions was less than a $15,000 standard deduction, took the standard deduction, and left out the MA income tax from SALT. It also used outdated brackets, and its submitted $39,827.88 contradicts its own $31,281.32."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly concluded that $22,543 of itemized deductions was less than the standard deduction and took the standard deduction. It never itemized the $31,027.41 that includes the $8,484.41 MA income tax, so taxable income was overstated at $160,402."
us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It computed a Massachusetts PFML employee share of 0.46% × $175,002 = $805.01, then dropped it, assuming a PolicyEngine-style payroll tax counts only Social Security and Medicare when state payroll tax is not modeled. It then submitted $13,268.70, which does not match even its own $13,387.65 subtotal. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($10,850.12) and Medicare ($2,537.53) and stopped at $13,387.65, federal tax alone, without mentioning Massachusetts PFML. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no separate mandatory employee payroll tax beyond federal taxes and did not count PFML. It capped Social Security at an outdated $168,600 wage base, though the engine taxes the full $175,002, and then submitted $13,383.41, which matches neither of its own subtotals. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It estimated the Massachusetts PFML employee contribution at about $0 and submitted $12,321.73, which contradicts its own $13,387.65 subtotal of Social Security and Medicare on full wages. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It set Massachusetts PFML to 0 for lack of a rate, then submitted $12,717.62, which is not the $13,387.65 sum of the Social Security and Medicare figures it had just computed. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It set Massachusetts PFML to about $0, wrote a total of about $13,388 and submitted $13,385, below the $13,387.65 of Social Security and Medicare. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($10,850.12) and Medicare ($2,537.53), named the Massachusetts PFML employee contribution (about $805 at 0.46%) and left it out, assuming the reference model does not include it, answering $13,387.65, federal tax alone. The engine does include it: the reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It first reached $14,192.66, the reference, with a 0.46% PFML employee rate, then switched to 0.604%: all of a 0.42% medical portion plus 40% of a 0.46% family portion. The engine uses 2026 rates of 0.18% family and 0.70% medical and counts all of the family rate and 40% of the medical, 0.46% of wages, the most Massachusetts's published 2026 rates let the employer deduct; its PFML came out at $1,057.01 against the reference's $805.01."
-us,scenario_081,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It treated Massachusetts PFML as employer-side only and left it out, then submitted $12,318.65, which contradicts its own $13,387.65 sum of Social Security and Medicare. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It said outright that it did not include any Massachusetts PFML contribution and answered $13,387.65, Social Security plus Medicare. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no employee state payroll tax and answered $13,387.65, Social Security plus Medicare. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It included a Massachusetts PFML employee share at 0.38% of wages, about $665, and rounded the total to $14,053. The reference counts the most Massachusetts's published 2026 rates let the employer deduct, 0.46% of $175,002 = $805.01, for $14,192.66."
-us,scenario_081,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It applied a 0.375% MA PFML employee rate ($656.26) instead of the 0.46% rate ($805.01). It also submitted $13,593.41, which differs from the $14,043.91 total stated in its own explanation."
-us,scenario_081,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It used a 0.44% MA PFML employee rate ($770.01) instead of 0.46% ($805.01), which put the total $35 short of $14,192.66."
-us,scenario_081,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It included Massachusetts PFML at an estimated 0.318% of wages ($556.51), for $13,944.16. The reference counts the most Massachusetts's published 2026 rates let the employer deduct, 0.46% of $175,002 = $805.01, so the total is $248.50 under it."
-us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It counted only 6.2% Social Security plus 1.45% Medicare and did not mention Massachusetts PFML. Its $11,843 does not even equal 7.65% × $175,002 = $13,387.65, the result of the formula it described. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It applied SS and Medicare to $167,613, having subtracted the $7,389 employer-sponsored insurance premiums, when the payroll tax base is the full $175,002. It also used an estimated PFML of about $580 instead of 0.46% x $175,002 = $805.01."
-us,scenario_081,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It cut the FICA base to $167,613 by subtracting the $7,389 ESI premiums, when SS and Medicare apply to the full $175,002 ($10,850.12 and $2,537.53). It also used a 0.24% PFML rate ($402) instead of 0.46% ($805.01)."
-us,scenario_081,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It included MA PFML at $765.01 instead of 0.46% x $175,002 = $805.01. It also miscomputed SS as $10,830 rather than 6.2% x $175,002 = $10,850.12, so the total came up $60 short."
-us,scenario_081,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It reduced FICA wages to $167,613 by subtracting the $7,389 employer-sponsored insurance premiums as pre-tax, when the engine's payroll tax base is the full $175,002, and it did not mention Massachusetts PFML. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It named Social Security, Medicare and Massachusetts PFML contributions, but its $13,444 total is only about $56 above Social Security plus Medicare on full wages ($13,387.65), so on full wages it counted about $56 of PFML. The reference counts the most Massachusetts's published 2026 rates let the employer deduct, 0.46% of $175,002 = $805.01, for $14,192.66."
-us,scenario_081,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It stopped at Social Security plus Medicare, $13,387.65, without mentioning Massachusetts PFML. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"It built a 0.532% PFML employee rate by taking 40% of the entire 0.88% combined rate (0.352%) as the medical share and adding the 0.18% family share. The engine counts 40% of only the 0.70% medical rate plus the 0.18% family rate, 0.28% + 0.18% = 0.46%, the most Massachusetts's published 2026 rates let the employer deduct; its PFML came out at $931 against the reference's $805.01."
-us,scenario_081,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no mandatory employee payroll tax and answered $13,387.65, Social Security plus Medicare. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare, submitting $13,389.37 rather than the $13,387.65 they come to, and did not mention Massachusetts PFML. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It wrongly claimed the wages exceed the $200,000 Additional Medicare Tax threshold ($175,002 does not) and submitted $5,420, less than half the $10,850.12 of Social Security tax alone, without mentioning Massachusetts PFML. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Its Social Security plus Medicare total of $13,387.65 matches the engine, but it estimated Massachusetts PFML withholding at $962.51, which is 0.55% of wages. The reference counts the most Massachusetts's published 2026 rates let the employer deduct, 0.46% of $175,002 = $805.01, for $14,192.66."
-us,scenario_081,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It counted only 6.2% Social Security and 1.45% Medicare, about $13,388, without mentioning Massachusetts PFML. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied the combined 2.9% employer-plus-employee Medicare rate instead of the 1.45% employee rate, doubling Medicare to about $5,075, and did not mention Massachusetts PFML. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It included Massachusetts PFML at 0.318% of wages ($556) and rounded the components to whole dollars, for $13,944. The reference counts the most Massachusetts's published 2026 rates let the employer deduct, 0.46% of $175,002 = $805.01, for $14,192.66."
-us,scenario_081,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It assumed there is no state payroll tax and applied only 7.65% FICA, rounded to $13,388. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"It applied a 0.256% MA PFML employee rate ($448.01) instead of 0.46% ($805.01), which put the total $357 short."
-us,scenario_081,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It returned $0 with no reasoning, as though the head owed no employee payroll tax at all. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It said Massachusetts levies no mandatory employee payroll tax modeled here and stopped at $13,387.65. The reference adds $805.01 (0.46% of $175,002), the most Massachusetts's published 2026 rates let the employer deduct for Massachusetts PFML. The law lets the employer deduct less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $13,387.65."
-us,scenario_081,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It said no mandatory state payroll taxes apply in Massachusetts, totaled $13,387.65, and then submitted an unexplained $14,525.24 that contradicts that total. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
-us,scenario_081,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It listed Social Security, Medicare and Additional Medicare Tax, not Massachusetts PFML, and submitted $13,922. By its own formula the total is 7.65% × $175,002 = $13,387.65 with no Additional Medicare Tax. The reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 PFML = $14,192.66, counting the most Massachusetts's published 2026 rates let the employer deduct for PFML (0.46% of $175,002); the law lets the employer deduct less or none, and without that share the total is $13,387.65."
+us,scenario_081,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It correctly worked out the PFML contribution as 0.46% × $175,002 = $805.01, then dropped it. Next it got the FICA-only total right at $13,387.65, but submitted $13,268.70, which matches none of its own components. The submitted value is an arithmetic and consistency error, and it also leaves out the PFML amount it had already calculated."
+us,scenario_081,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($10,850.12) and Medicare ($2,537.53) correctly but left out the Massachusetts PFML employee contribution. The reference counts that contribution at 0.46% × $175,002 = $805.01. Its $13,387.65 is the answer under the reading that the employer absorbs the optional employee PFML share."
+us,scenario_081,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It used the 2024 Social Security wage base of $168,600 as the 2026 cap, which cut Social Security tax to $10,453.20. The 2026 base is above $175,002, so the full wages are taxable ($10,850.12). It also said Massachusetts has no employee payroll tax and left out the $805.01 PFML contribution. Its final $13,383.41 doesn't match any of its own subtotals."
+us,scenario_081,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It counted only federal FICA (6.2% + 1.45% on $175,002, rounded to $13,388) and left out the $805.01 Massachusetts PFML employee contribution at 0.46%. That matches the reading where the employer pays the employee's PFML share."
+us,scenario_081,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It calculated FICA at $13,387.65 and then submitted $12,321.73, explaining it with a nonexistent cap even though it had just said all $175,002 of wages fall under the wage base. It also misread PFML as roughly $0 for the employee. The actual employee share is 0.18% family plus 40% of the 0.70% medical rate, which is 0.46%, or $805.01."
+us,scenario_081,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added $10,850.12 + $2,537.53, which is $13,387.65, but submitted $12,717.62, an addition error of $670. It also set the Massachusetts PFML employee contribution to $0 instead of using the 0.46% employee rate ($805.01)."
+us,scenario_081,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It treated the Massachusetts PFML employee contribution as about $0 and submitted FICA only. It also made a small slip, entering $13,385 instead of $13,387.65. The $805.01 gap to the reference is the PFML contribution at 0.46%, which it left out under the reading that the employer pays that share."
+us,scenario_081,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It correctly calculated the Massachusetts PFML employee contribution as $805.01 at 0.46%, then chose to leave it out and submitted FICA only ($13,387.65). That is the reading where the employer pays the optional employee PFML share, while the reference includes it."
+us,scenario_081,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It first reached the reference value of $14,192.66 using 0.46%, then switched to a made-up rate split: family 0.46%, medical 0.42%, with the employee paying all of medical plus 40% of family, for 0.604%. The actual 2026 rates are 0.18% family, which the employee can be charged in full, and 0.70% medical, of which the employee can be charged 40%, for a total employee share of 0.46%. Its PFML amount of $1,057.01 is $252 too high."
+us,scenario_081,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It called Massachusetts PFML an employer-only charge and left out the $805.01 employee share at 0.46%. It then submitted $12,318.65 even though its own total was $13,387.65, an addition error of $1,069."
+us,scenario_081,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly at $13,387.65 and explicitly left out the Massachusetts PFML employee contribution. The reference includes that contribution at 0.46% × $175,002 = $805.01, using the maximum share the employer can withhold."
+us,scenario_081,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no employee state payroll tax and so missed the PFML employee contribution of 0.46% × $175,002 = $805.01. Its FICA-only $13,387.65 matches the reading where the employer absorbs the employee PFML share."
+us,scenario_081,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It included Massachusetts PFML but used a 0.38% employee rate instead of 0.46%. The correct rate is the full 0.18% family rate plus 40% of the 0.70% medical rate, and the wrong rate understated PFML by $140 ($665 instead of $805.01)."
+us,scenario_081,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It used an outdated PFML employee rate of 0.375% ($656.26) instead of 0.46% ($805.01). It also submitted $13,593.41, which doesn't match its own stated total of $14,043.91."
+us,scenario_081,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It used a 0.44% PFML employee rate ($770.01) instead of 0.46%. The correct rate is 0.18% family plus 40% of the 0.70% medical rate, or $805.01, so it came out $35 short."
+us,scenario_081,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It used the outdated 0.318% PFML employee rate ($556.51) instead of the 2026 employee share of 0.46%. The 2026 share is 0.18% family plus 40% of the 0.70% medical rate, which is $805.01."
+us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"6.2% + 1.45% of $175,002 is $13,387.65, but it submitted $11,843, an arithmetic error of more than $1,500. It also left out the $805.01 Massachusetts PFML employee contribution at 0.46%."
+us,scenario_081,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It subtracted the $7,389 employer-sponsored insurance premiums from the FICA wage base and taxed $167,613. The engine applies Social Security, Medicare, and PFML to the full $175,002 of gross wages. Its PFML add-on of about $580 is also below the correct $805.01 at 0.46%."
+us,scenario_081,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It taxed $167,613 for FICA, which is wages minus the $7,389 ESI premiums, when the base should be the full $175,002. That understated Social Security by $458 and Medicare by $107. It also used a 0.24% PFML rate ($402) instead of 0.46% on $175,002 ($805.01)."
+us,scenario_081,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It entered Social Security as $10,830 instead of 6.2% × $175,002 = $10,850.12. It also entered PFML as $765.01, about 0.437%, instead of 0.46% × $175,002 = $805.01. Together these undercount the total by about $60."
+us,scenario_081,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It lowered the FICA base to $167,613 by subtracting the $7,389 employer-sponsored insurance premiums as pre-tax deductions, while the engine applies FICA to the full $175,002 of gross wages. It also left out the $805.01 Massachusetts PFML employee contribution."
+us,scenario_081,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Its $13,444 total implies only about $56 of PFML on top of the $13,387.65 FICA. The correct employee PFML contribution is 0.46% × $175,002 = $805.01, so it badly understated the Massachusetts employee rate."
+us,scenario_081,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare ($13,387.65) and left out the Massachusetts PFML employee contribution. The reference includes that contribution at the maximum withholdable 0.46% share, $805.01."
+us,scenario_081,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"It built a PFML employee rate of 0.532% from a 0.352% medical share plus 0.18% family. The employee medical share is actually 40% of 0.70%, or 0.28%, so the correct total is 0.46% ($805.01), not $931."
+us,scenario_081,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no mandatory employee payroll tax and submitted FICA only ($13,387.65). That leaves out the PFML employee contribution of 0.46% × $175,002 = $805.01, which the reference includes as the maximum share the employer can withhold."
+us,scenario_081,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It counted only federal FICA and left out the $805.01 Massachusetts PFML employee contribution at 0.46%. It also made a $1.72 arithmetic slip, giving $13,389.37 instead of $13,387.65."
+us,scenario_081,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It wrongly said $175,002 of wages is above the $200,000 Additional Medicare Tax threshold. It then submitted $5,420, well below even the $10,850.12 Social Security component. The correct total is FICA of $13,387.65 plus $805.01 of PFML."
+us,scenario_081,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It estimated PFML withholding at $962.51, about 0.55% of wages, instead of the 0.46% employee share. The correct share is 0.18% family plus 40% of the 0.70% medical rate, or $805.01, so PFML was overstated by $157.50."
+us,scenario_081,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare (about $13,388) and left out the Massachusetts PFML employee contribution of 0.46% × $175,002 = $805.01 that the reference includes."
+us,scenario_081,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied the combined 2.9% Medicare rate (employer plus employee) instead of the 1.45% employee rate: $175,002 × 9.1% ≈ $15,926. It also left out the $805.01 Massachusetts PFML employee contribution."
+us,scenario_081,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used the outdated 0.318% PFML employee rate ($556) instead of the 2026 employee share of 0.46%. That share is 0.18% family plus 40% of the 0.70% medical rate, or $805.01."
+us,scenario_081,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $175,002 and said there is no state payroll tax, which leaves out the Massachusetts PFML employee contribution of 0.46% ($805.01) that the reference includes."
+us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there was nothing to compare with the $14,192.66 reference."
+us,scenario_081,payroll_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a 0.256% PFML employee rate ($448.01) instead of 0.46%. The correct rate is the full 0.18% family rate plus 40% of the 0.70% medical rate, which is $805.01."
+us,scenario_081,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It submitted $0 even though the household has $175,002 of wages, so it skipped the $10,850.12 Social Security, the $2,537.53 Medicare, and the $805.01 Massachusetts PFML employee contribution."
+us,scenario_081,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly at $13,387.65 but said Massachusetts has no mandatory employee payroll tax. That leaves out the PFML employee contribution of 0.46% × $175,002 = $805.01 that the reference includes."
+us,scenario_081,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It calculated $13,387.65 for FICA, said Massachusetts has no employee payroll tax, and then submitted an unrelated $14,525.24 that it called a 'rounded' result. It also never included the $805.01 PFML employee contribution at 0.46%."
+us,scenario_081,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It cited Additional Medicare Tax, which is $0 because wages are under $200,000, and never mentioned Massachusetts PFML. Its $13,922 is the $13,387.65 FICA plus an unexplained $534, not the 0.46% PFML contribution of $805.01."
us,scenario_081,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached about $8,238: $164,602 of Part B at 5% ($8,230.10) plus $166 of interest and dividends at 5% ($8.30), which matches the correct total. It then added an unexplained ~$142 'rounding of components' and submitted $8,380. No Massachusetts rule supports that addition."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $4,400 exemption, the $4,000 rent deduction and the $2,000 payroll deduction correctly. However, it used the $1,080 short-term loss to wipe out the $166 of interest and dividends. The $56 of interest belongs in Part B income ($175,058) and the $110 of dividends stays taxable in Part A. Its answer is therefore missing $2.80 plus $5.51 of tax: $8,232.90 + $5.51 = $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted an invented $4,550 deduction and skipped the $2,000 payroll deduction, the $4,000 rent deduction and the $4,400 personal exemption. It then took off $3,835 of made-up nonrefundable credits (a dependent exemption credit and a property tax credit), even though there are no dependents and no Massachusetts nonrefundable credit applies. The correct figure is 5% of $164,658 in Part B plus 5% of the $110 of dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full ~$11,287 of FICA, but the Massachusetts Social Security/Medicare deduction is capped at $2,000. It also refused the rent deduction on the grounds that the head owns a home, even though $44,400 of rent is listed, which gives the capped $4,000 deduction. It then inflated its own $7,966 to $8,730 with no basis."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from $175,168 of income. It left out the $2,000 Social Security/Medicare deduction and the $4,000 rent deduction (50% of $44,400, capped). Those two deductions bring Part B taxable income down to $164,658."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $4,400 exemption and the $2,000 payroll deduction but left out the $4,000 rent deduction for the $44,400 of rent. It then pushed its own $8,438 up to $8,620 with unexplained 'small adjustments'."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at $3,000 instead of the current $4,000. It also left out the $2,000 Social Security/Medicare deduction entirely and added the $6 of REIT income to the 5% base. Together these overstated taxable income by $3,006 ($167,774 against the correct $164,658 plus $110)."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the head as a homeowner and denied the rent deduction, even though $44,400 of rent gives the capped $4,000 deduction. It also skipped the $2,000 payroll-tax deduction, which produced $8,539. It then cut that to $8,200 with an unexplained 'exemption differences' adjustment."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from $174,088, which nets the $1,080 short-term capital loss against wage income; Massachusetts does not allow that loss against Part B income. It also left out the $2,000 Social Security/Medicare deduction, which gave $165,688 of taxable income instead of $164,658 in Part B plus $110 in Part A."
-us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used federal AGI ($174,094, which nets the capital loss) as the base and a $4,800 exemption instead of the $4,400 single exemption. It left out both the $4,000 rent deduction and the $2,000 payroll-tax deduction."
-us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $22,543 of federal-style itemized deductions (mortgage interest and taxes), which Massachusetts does not allow. It also used a wrong AGI of $166,705 and a $5,000 exemption instead of $4,400, and it never applied the $4,000 rent or $2,000 payroll deductions correctly against $175,058 of Part B income."
-us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It correctly applied the rent, payroll and exemption deductions, but it also subtracted $22,543 of federal itemized deductions (mortgage interest and real estate taxes), which Massachusetts does not allow. That undercounted taxable income by more than $20,000. Its explanation reports $7,111.55, which does not match the $7,519.53 it submitted."
-us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $16,900 of mortgage interest and $2,000 of real estate taxes as if they were federal itemized deductions, which Massachusetts does not allow. It also left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction that make up the correct Part B base."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at $3,000 instead of $4,000, which overstates tax by $50. It also left the $56 of interest out of Part B and never taxed the $110 of Part A dividends. The correct result is $164,658 × 5% + $5.51 = $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to about $175,000 of gross income and took no deductions. It skipped the $4,400 personal exemption, the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, which together bring Part B taxable income down to $164,658."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It took the $7,389 of employer-sponsored insurance premiums out of wages. The input is gross wages, so Part B income stays at $175,002 plus $56 of interest. As a result its Part B base of $157,219 is about $7,400 too low, even though the exemption, rent and payroll deductions were right."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by subtracting the $7,389 of ESI premiums from the listed gross wages, which lowered Part B income. It then left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, applying only the $4,400 exemption."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $8,750 equals 5% of $175,000 of gross wages with no deductions, so it skipped the $4,400 exemption, the $4,000 rent deduction and the $2,000 payroll deduction. With those applied, Part B taxable income is $164,658, and the correct tax including $110 of Part A dividends is $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by taking the $7,389 of ESI premiums out of the listed gross wages, which understated Part B income by $7,389. It also dropped the $56 of Part B interest and the $110 of Part A dividends. Its $4,400, $4,000 and $2,000 deductions were right."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,869 equals 5% of about $157,380: wages reduced by the $7,389 of ESI premiums, less $4,400, $4,000 and $2,000, plus $166 of interest and dividends. Taking the premiums out of the listed gross wages understated Part B income by $7,389."
-us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $4,400 single personal exemption and the $2,000 Social Security/Medicare deduction, and it used the old $3,000 rent cap instead of $4,000. It also netted the $1,080 short-term loss against ordinary income. The result was $171,094 of taxable income against the correct $164,658 plus $110."
-us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the old $3,000 rent-deduction cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction entirely. It also added the $6 of REIT income to the base, which overstated taxable income by $3,006."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $6,440 implies about $128,800 of taxable income, roughly $36,000 below the correct $164,768. That gap matches subtracting federal-style itemized deductions (the $16,900 of mortgage interest and state and local taxes), which Massachusetts does not allow. Only the $2,000 payroll, $4,000 rent and $4,400 exemption deductions reduce the $175,058 of Part B income."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Its $13,951 is about 8.5% of the roughly $164,000 taxable base, which is Massachusetts's short-term capital gains rate. Part B wage income and Part A dividends are taxed at the flat 5%, which gives $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $2,000 payroll, $4,000 rent and $4,400 exemption deductions correctly. However, it let the $1,080 short-term loss offset the $166 of interest and dividends. The $56 of interest belongs in Part B income ($175,058) and the $110 of dividends stays taxable at 5% in Part A, which adds $2.80 + $5.51 to reach $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It cited only the exemption and the rent deduction and left out the $2,000 Social Security/Medicare deduction. On $175,002 of wages those two deductions alone give $8,330, so its $7,915 includes about $8,300 of unsupported extra deductions and also leaves out the Part A dividend tax. The correct base is $175,058 − $2,000 − $4,000 − $4,400 = $164,658 in Part B plus $110 in Part A."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,338.70 is 5% of $166,774: $175,174 less $4,400 and $4,000. It left out the $2,000 Social Security/Medicare deduction and added the $6 of REIT income to the base."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $56 of interest with a bank-interest exemption and used the short-term capital loss to cancel the $110 of dividends. The interest stays in Part B income ($175,058) and the dividends are taxed at 5% in Part A ($5.51). With its correct $2,000, $4,000 and $4,400 deductions, the result is $8,232.90 + $5.51 = $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,284 is 5% of about $165,688: income net of the $1,080 short-term loss, less the $4,400 exemption and the $4,000 rent deduction. It left out the $2,000 Social Security/Medicare deduction and wrongly netted the short-term capital loss against Part B income."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $2,000 Social Security/Medicare deduction, which overstates tax by $100. It also used the short-term loss to cancel the $56 of interest, which belongs in Part B, and the $110 of dividends, which Part A taxes at 5%."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $56 of bank interest as exempt and let the capital loss absorb the $110 of dividends. The interest stays in Part B income ($175,058) and the dividends are taxed at 5% in Part A ($5.51). With its correct $2,000, $4,000 and $4,400 deductions, the result is $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to the full $174,094 of federal AGI. It skipped the $4,400 personal exemption, the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it netted the capital loss against wage income."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $1,782 of unreimbursed employee business expenses, which PolicyEngine's Part B base does not include. It left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it netted the $1,080 short-term loss against ordinary income."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $4,000 rent deduction (50% of $44,400, capped) and subtracted a made-up $1,000 of capital loss from ordinary income. It also added the $6 of REIT income, which gave $167,774 of taxable income instead of $164,658 in Part B plus $110 in Part A."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI of $174,094, which nets the $1,080 short-term loss against wages, and subtracted only the $4,400 exemption and the $4,000 rent deduction. It left out the $2,000 Social Security/Medicare deduction that Massachusetts allows against Part B income."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from federal AGI of $174,094. It left out the $4,000 rent deduction for the $44,400 of rent and the $2,000 Social Security/Medicare deduction."
-us,scenario_081,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the old $3,000 rent-deduction cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction. It also started from federal AGI, which nets the capital loss against wage income."
-us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there is no answer to score. The correct figure is 5% of $164,658 of Part B taxable income plus 5% of $110 of Part A dividends, which is $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It said there was 'no separate rent expense input', but the facts list $44,400 of pre-subsidy rent, which gives the capped $4,000 rent deduction. It also started from $174,088, which nets the $1,080 short-term loss against Part B income."
-us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned zero Massachusetts tax with no reasoning, even though a single filer with $175,002 of wages owes the 5% flat tax. The correct figure is 5% × ($175,058 − $2,000 − $4,000 − $4,400) + 5% × $110 = $8,238.41."
-us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It reduced $175,002 of wages only by the $4,400 exemption. It left out the $4,000 rent deduction for the $44,400 of rent and the $2,000 Social Security/Medicare deduction, which overstated Part B taxable income by $6,000."
-us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It said Massachusetts allows no deductions other than the personal exemption and applied 5% to $174,094 less $4,400. That misses the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it nets the short-term loss against wages."
-us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,800 implies about $156,000 of taxable income, roughly $8,800 below the correct $164,768 ($164,658 in Part B plus $110 in Part A). That means it subtracted items beyond the only allowed deductions: the $2,000 payroll deduction, the $4,000 rent deduction and the $4,400 exemption, taken from $175,058."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own working was effectively correct: $164,602 of Part B income at 5% is $8,230.10, plus $8.30 on interest and dividends, for about $8,238. It then raised the figure to $8,380 with a made-up 'rounding of components' adjustment and submitted a number its own work didn't support."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It got the wage-side deductions right ($2,000 FICA, $4,000 rent, $4,400 exemption). But it let the short-term capital loss cancel all the interest and dividends. The $56 of interest belongs in Part B, adding $2.80 of tax, and the dividends stay taxable in Part A, adding about $5.51; dropping both left it $8.41 short."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It made up $3,835 of nonrefundable 'property tax and personal exemption' credits. A single filer with no dependents who is under 65 and earns $175k gets none of them. It also never subtracted the $4,400 personal exemption, the $4,000 rent deduction or the $2,000 FICA deduction from Part B income."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It submitted a rounded $8,000 without computing Part B. The correct steps are $175,058 less the $2,000 FICA deduction, the $4,000 rent deduction and the $4,400 exemption, giving $164,658 at 5% plus about $5.51 of Part A dividend tax, for $8,238."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $11,287 of FICA, but Massachusetts caps that deduction at $2,000. It also skipped the renter deduction by claiming the head owns the home, even though $44,400 of rent is listed, which earns the $4,000 maximum. It then pushed its $7,966 result up to $8,730 with no basis."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 personal exemption. It left out the $2,000 Social Security/Medicare deduction and the $4,000 renter deduction, so its $8,538 is exactly $300 (5% of $6,000) too high."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the exemption and the $2,000 FICA deduction but never applied the $4,000 renter deduction, even with $44,400 of rent listed. It then added unexplained 'adjustments' that took its $8,438 up to $8,620."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at the pre-2023 $3,000 instead of the current $4,000. It also never took the $2,000 deduction for Social Security/Medicare withholding, which overstated taxable income by $3,000 ($150 of tax)."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $8,539 using only the personal exemption, leaving out the $4,000 renter deduction and the $2,000 FICA deduction. It then cut the figure to $8,200 based on unexplained 'minor exemption differences' instead of applying those deductions."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It never took the $2,000 Part B deduction for Social Security/Medicare withholding. It also started from $174,088, which nets the $1,080 short-term capital loss against wage income; Massachusetts doesn't allow that, and the correct Part B base is $175,058."
+us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $4,800 exemption instead of the $4,400 single exemption. It also never applied the $4,000 renter deduction or the $2,000 FICA deduction, and it started from federal AGI, which nets the short-term loss against wages."
+us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $22,543 of federal-style itemized deductions (mortgage interest and real estate taxes), which Massachusetts doesn't allow. It also used a made-up $5,000 exemption and an understated $166,705 AGI, and it never applied the $4,000 rent or $2,000 FICA deductions."
+us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It stacked $22,543 of federal itemized deductions (mortgage interest and property taxes) on top of the correct rent, FICA and exemption deductions. Massachusetts has no itemized deduction for those items. The $7,519.53 it submitted doesn't even match the $7,111.55 its own reasoning produced."
+us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $16,900 of mortgage interest and $2,000 of real estate taxes, but Massachusetts Part B income allows neither. It also left out the $4,000 renter deduction and the $2,000 FICA deduction that do apply."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It capped the renter deduction at the old $3,000 instead of the $4,000 cap in effect since 2023. It also left out the $56 of interest in Part B and the Part A tax on the $110 of dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to roughly $175,000 of gross income with no $4,400 personal exemption, no $4,000 renter deduction and no $2,000 FICA deduction. The correct Part B taxable income is $164,658."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,389 of employer-sponsored insurance premiums from the $175,002 of wages, but those wages are the full Massachusetts Part B wage base. That understated taxable income by about $7,400 even though the rent, FICA and exemption deductions were right."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by subtracting the $7,389 of employer insurance premiums from wages, which shouldn't be reduced. It then left out the $4,000 renter deduction and the $2,000 FICA deduction, so the two mistakes partly offset each other."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It taxed roughly $175,000 at 5% with no personal exemption, renter deduction or FICA deduction. Those three subtractions total $10,400 and bring Part B taxable income down to $164,658."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from 'taxable wages' of $167,613 by subtracting the $7,389 of employer-sponsored insurance premiums from wages. The full $175,002 (plus $56 of interest) is the Part B base, so taxable income came out $7,445 too low."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,869 equals 5% of about $157,380. That is consistent with subtracting the $7,389 of employer-sponsored insurance premiums from wages before the otherwise-correct exemption, rent and FICA deductions, but Part B wages shouldn't be reduced by those premiums."
+us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $4,400 personal exemption or the $2,000 FICA deduction, and it used the outdated $3,000 rent cap instead of $4,000. It also netted the short-term capital loss against wage income, which Massachusetts Part B doesn't allow."
+us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-2023 $3,000 rent-deduction cap instead of $4,000. It also never took the $2,000 deduction for Social Security/Medicare withholding, which overstated taxable income by $3,000."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $6,440 implies about $128,800 of taxable income, roughly $36,000 below the correct $164,658 Part B plus about $110 Part A. That gap is consistent with subtracting federal itemized deductions (mortgage interest and property taxes), which Massachusetts doesn't allow."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Its $13,951 implies about $279,000 taxed at 5%, far above the household's $175,058 of Part B income. It didn't apply the 5% rate to $164,658, which is what's left after the $2,000 FICA, $4,000 rent and $4,400 exemption deductions."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It got the Part B wage deductions right but used the short-term capital loss to wipe out all $166 of interest and dividends. The $56 of interest is Part B income, adding $2.80 of tax, and the dividends stay taxable in Part A, adding about $5.51."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,915 implies about $158,300 of taxable income, roughly $6,500 below the correct $164,658. Since only the exemption and rent deduction were claimed, that figure requires an extra exclusion from wages, consistent with removing the $7,389 of employer insurance premiums."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It applied the exemption and the $4,000 renter deduction but left out the $2,000 Part B deduction for Social Security/Medicare withholding. That's 5% of $2,000, which accounts for the $100 overstatement."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $56 of interest as exempt and let the capital loss absorb the dividends. The interest is taxed in Part B ($2.80) and the dividends stay taxable in Part A (about $5.51), so it fell $8.31 short."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It applied only the personal exemption and the renter deduction, leaving out the $2,000 deduction for Social Security/Medicare withholding. It also netted the short-term loss into its income base, which Massachusetts Part B doesn't allow."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $4,400 exemption and $4,000 rent deduction from wages but not the $2,000 FICA deduction. It also dropped the taxable $56 of interest (Part B) and the $110 of dividends (Part A)."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the bank interest as exempt and let the capital loss absorb the dividends. The $56 of interest is taxed in Part B and the dividends in Part A, adding about $8.31 to its $8,230.10."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% straight to federal AGI of $174,094, skipping the $4,400 personal exemption, the $4,000 renter deduction and the $2,000 FICA deduction that bring Part B taxable income to $164,658."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $1,782 of unreimbursed employee business expenses, which Massachusetts Part B doesn't allow. It also left out the $4,000 renter deduction and the $2,000 FICA deduction that do apply."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It never applied the $4,000 renter deduction even though $44,400 of rent is listed. It also subtracted a made-up $1,000 capital-loss allowance from ordinary income."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $2,000 Part B deduction for Social Security/Medicare withholding. It also started from $174,094 of federal AGI, which nets the short-term capital loss against wages instead of using the $175,058 Part B base."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from AGI, leaving out the $4,000 renter deduction for the $44,400 of rent and the $2,000 FICA deduction."
+us,scenario_081,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the outdated $3,000 rent cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction. It also started from federal AGI, which nets the short-term loss against wages."
+us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It stated there was 'no separate rent expense input' despite $44,400 of listed pre-subsidy rent, so it left out the $4,000 renter deduction. It also netted the capital loss into the $174,088 base instead of using $175,058 of Part B income."
+us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $0 with no reasoning, even though a single resident with $175,002 of wages owes Massachusetts' 5% tax on $164,658 of Part B income plus Part A tax on the dividends, $8,238.41 in total."
+us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from wages, leaving out the $4,000 renter deduction and the $2,000 deduction for Social Security/Medicare withholding. That's a $6,000 overstatement of the base, or $300 of tax."
+us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly said Massachusetts allows no relevant deductions and applied only the $4,400 exemption, leaving out the $4,000 renter deduction and the $2,000 FICA deduction. It also netted the short-term loss against wages."
+us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,800 implies about $156,000 of taxable income, roughly $8,800 below the correct $164,658 Part B plus about $110 Part A. That reflects deductions beyond the $2,000 FICA, $4,000 rent and $4,400 exemption, which are the only Part B subtractions available."
us,scenario_081,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 1 as within the applicable CHIP age range. The trace applies CHIP's age criterion against the child and returns false, while Medicaid eligibility also returns NONE."
us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested child1_chip_eligible output and supplied no explanation, violating the required submission contract."
@@ -6696,52 +6870,52 @@ us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,health_coverage,Fa
us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household."
us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Submitted $14,748, which contradicts its own worked result of about $10,353. That worked chain also skipped the $700 2026 non-itemizer charitable deduction (taxable income $93,436 instead of $92,735.15). It also used HoH thresholds of $17,300/$65,900 instead of the 2026 $17,700/$67,450, and applied a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its ordinary tax of $10,444.60 on $79,380 is $80 too high: the 2026 HoH schedule (10% to $17,700, 12% to $67,450, 22% above) gives $10,364.60. It also added the $68 state tax refund to AGI. The head takes the standard deduction, so the refund produced no prior-year tax benefit and is excluded."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Filed as single with a $14,600 standard deduction instead of head of household with the $24,150 2026 HoH standard deduction. It netted the whole $30,711 capital loss against income instead of capping the deduction at $3,000, and figured the CDCC on the full $9,600 instead of the $3,000 one-child cap. It then submitted $23,437, a figure unconnected to its own tax estimate of about $8,506."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used a $23,500 HoH standard deduction instead of $24,150 and omitted the $700 non-itemizer charitable deduction. It also taxed ordinary income on stale thresholds (10% to $17,000, 12% to $64,850) instead of the 2026 $17,700/$67,450. That gave $10,922 of ordinary tax on $80,670, versus $10,349.45 on the correct $79,311.15."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its own chain reached $10,460, and it then raised the answer to $11,150 with no stated reason. That chain used a $23,000 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund. It also used stale bracket thresholds ($17,000/$64,850 instead of $17,700/$67,450) and a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income on stale HoH thresholds (10% to $17,000, 12% to $64,850) instead of the 2026 $17,700/$67,450. That produced $10,716 on $79,730, where the 2026 schedule gives $10,441.60. It also used a $24,500 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund to income."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its only error is adding the $68 state tax refund to income. The head claims the standard deduction, so the refund carried no prior-year tax benefit and is excluded, leaving taxable income of $92,735.15. The extra $68.85 of taxable income at 22% accounts for its $15.15 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Left the $13,424 of qualified dividends out of AGI ($106,519) and then subtracted them again from taxable income, so ordinary taxable income was $70,595 instead of $79,311.15. It also added the $2,289 short-term gain as separate ordinary income after already netting it into the capital loss. On top of that it used a $22,500 standard deduction, stale brackets, and a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Submitted $13,920 even though its own chain ended at $10,739. That chain used a $22,500 standard deduction instead of $24,150 and omitted the $700 non-itemizer charitable deduction. It also used 2024 HoH thresholds ($16,550/$63,100) instead of the 2026 $17,700/$67,450, and a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the extra $700 charitable deduction that non-itemizers get starting in 2026. It also added the $68 state refund to income and dropped the $8 IRA deduction as phased out. That $776.86 of excess taxable income at 22% accounts for its $171 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the full $33,000 long-term loss and then another $3,000, instead of netting to a $30,711 loss and capping the deduction at $3,000, which cut AGI to $86,949. It also used a nonexistent $46,000 HoH standard deduction instead of $24,150. It computed the CDCC as 35% of $5,000 instead of 20% of the $3,000 one-child cap, and limited the nonrefundable CTC to $300 instead of the full $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Applied pre-TCJA law ($9,680 of personal exemptions, 10/15/25% brackets, a $1,000 CTC), but the OBBBA made the TCJA structure permanent for 2026. It also itemized $26,228 under those pre-TCJA rules. The correct 2026 path has no exemptions, the $24,150 standard deduction plus the $700 non-itemizer charitable deduction, 10/12/22% brackets, and a $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired: it applied $11,000 of personal exemptions, pre-TCJA brackets, and a $1,000 CTC that it phased out entirely. The 2026 regime, made permanent by the OBBBA, has no exemptions and a $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. It also gives the full $2,200 CTC, since income is below the $200,000 phase-out threshold."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used a $23,100 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund. Its thresholds were below the 2026 HoH schedule, giving $11,059.60 on $81,130 where 10% to $17,700 and 12% to $67,450 give $10,749.60. It also applied a $2,300 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Applied pre-TCJA law: $10,000 of personal exemptions, $18,564 of uncapped SALT as an itemized deduction, and a $1,000 CTC. Under 2026 law there are no exemptions, and the $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction beats itemizing. The 2026 CTC is $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Reported AGI of $100,571, about $17,000 below the correct $117,585.14. The correct AGI is wages net of the 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capped loss and the $8 IRA deduction. It subtracted only a $2,000 CTC and omitted the $600 CDCC and the 2026 CTC amount of $2,200, and its implied $13,093 pre-credit tax does not follow from the 2026 HoH brackets on its own AGI."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $21,208 of employer-sponsored insurance premiums from wages (AGI $96,446), but employment income stays at $100,013.62 after only the 401(k) deferral. It also applied pre-TCJA rules: $10,400 of personal exemptions, itemizing $14,298, and a fully phased-out CTC. As a result it subtracted only the $600 CDCC and dropped the $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $21,208 of employer-sponsored insurance premiums from wages (AGI $96,454), and applied pre-TCJA personal exemptions and a $1,000 CTC. The 2026 computation keeps employment income at $100,013.62 after only the 401(k) deferral and has no personal exemptions. It allows the $700 non-itemizer charitable deduction and a $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. Its $12,790.25 sits just above the correct pre-credit tax of $12,363.05 and $3,227 above the after-credit amount. That is the pattern of a regular-tax figure that never subtracted the $2,200 CTC and the $600 CDCC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted personal exemptions, which the OBBBA permanently eliminated for 2026, understating taxable income and landing $2,183 below the correct amount. The correct taxable income is $92,735.15: AGI of $117,585.14 less the $24,150 HoH standard deduction and the $700 non-itemizer charitable deduction."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. Its $3,384 tracks deducting the full $30,711 net capital loss instead of the $3,000 §1211(b) cap, which drops taxable income to about $65,000 and puts the qualified dividends in the 0% bracket. The correct chain taxes $92,735.15: $10,349.45 on ordinary income plus $2,013.60 on dividends, less $2,800 of credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted a $2,000 CTC instead of the 2026 $2,200. Its implied pre-credit tax of $12,855 is $492 above the correct $12,363.05. It skipped the $700 non-itemizer charitable deduction and used HoH bracket thresholds below the 2026 $17,700/$67,450."
-us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Used an estimated $23,063 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and dropped the $8 IRA deduction. It used bracket thresholds of $16,913/$64,575 instead of the 2026 $17,700/$67,450, and applied a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $21,208 of employer-sponsored insurance premiums as deductible medical expenses and itemized $23,266. It used that total even though it is below the $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. It also taxed ordinary income on thresholds below the 2026 schedule, giving $11,046 on $80,972 instead of $10,714.84."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $12,719 is $3,156 above the correct $9,563.05 and close to the correct pre-credit tax of $12,363.05. Although it says it applied them, it never subtracted the $2,200 CTC and $600 CDCC from regular tax."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Itemized real estate taxes, medical and OTC costs, and charity instead of taking the larger $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. Its $17,303 exceeds even the correct pre-credit tax of $12,363.05, so it also failed to subtract the $2,800 of CTC and CDCC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized $24,415 instead of taking the $24,150 standard deduction plus the $700 non-itemizer charitable deduction ($24,850 total). It also added the $68 refund, dropped the $8 IRA deduction, and subtracted a $2,000 CTC instead of the 2026 $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the entire $30,711 net capital loss (AGI $89,951 = $100,014 + $20,480 + $100 + $68 − $30,711) instead of capping the deduction at $3,000 under §1211(b). That understated AGI by about $27,600 and dropped the tax far below $9,563.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the extra $700 charitable deduction that non-itemizers get starting in 2026. It also added the $68 refund to AGI ($117,662) and dropped the $8 IRA deduction. That $776.86 of excess taxable income at 22% produces its $171 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its implied pre-credit tax of $13,036 is $673 above the correct $12,363.05, an overstatement of ordinary tax. It skipped the $700 non-itemizer charitable deduction and applied bracket thresholds below the 2026 HoH schedule (10% to $17,700, 12% to $67,450)."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Itemized roughly $28,500 of medical, charitable and SALT deductions by treating the $21,208 of employer-sponsored insurance premiums as deductible medical expenses. Without those premiums, itemized deductions fall well below the $24,150 standard deduction. The correct path takes the standard deduction plus the $700 non-itemizer charitable deduction, for taxable income of $92,735.15."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the $700 non-itemizer charitable deduction available from 2026. It also included the $68 refund and dropped the $8 IRA deduction, giving taxable income of $93,512 instead of $92,735.15 and a $171 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used the $24,150 standard deduction but skipped the $700 non-itemizer charitable deduction and added the $68 refund, giving taxable income of $93,504 instead of $92,735.15. The $768.85 excess at 22% explains its $169.15 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Itemized roughly $28,400 of medical, charitable and SALT deductions by counting the $21,208 of employer-sponsored insurance premiums as medical expenses. Without them, itemizing loses to the $24,150 standard deduction. The correct path takes the standard deduction plus the $700 non-itemizer charitable deduction, for taxable income of $92,735.15."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation, and its round $14,000 exceeds even the correct pre-credit tax of $12,363.05. It therefore neither derived the $92,735.15 of taxable income nor subtracted the $2,200 CTC and $600 CDCC."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA sunset: a pre-TCJA standard deduction, two personal exemptions, and a CTC fully phased out. It also subtracted the $21,208 of employer-sponsored premiums from wages (AGI $96,454). Under 2026 law there are no exemptions, employment income stays at $100,013.62 after the 401(k) deferral, and the full $2,200 CTC applies."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset law: two $5,300 exemptions, 10/15/25% brackets, and a $1,000 CTC phased out above a $75,000 threshold. Under the permanent 2026 regime there are no exemptions and brackets are 10/12/22% (to $17,700/$67,450). The standard deduction plus the $700 non-itemizer charitable deduction beats its $15,465 itemized total, and the full $2,200 CTC applies."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Itemized $28,094 by counting the $21,208 of employer-sponsored premiums as medical expenses above the 7.5% floor, alongside property tax and NY income tax. Without those premiums, itemized deductions fall below the $24,150 standard deduction plus the $700 non-itemizer charitable deduction. It therefore taxed $89,500 instead of $92,735.15."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration, using a $12,591 standard deduction and pre-TCJA brackets for $15,170 of ordinary tax on $105,063 of taxable income, and subtracted no CTC. 2026 law gives a $24,150 HoH standard deduction plus the $700 charitable deduction and 10/12/22% brackets. It also allows a $2,200 CTC alongside the $600 CDCC."
-us,scenario_082,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Omitted the $700 non-itemizer charitable deduction and added the $68 refund, giving taxable income of $93,504 instead of $92,735.15. Its ordinary tax of about $10,638 on $80,080 is about $120 above the $10,518.60 that the 2026 HoH schedule (10% to $17,700, 12% to $67,450) produces."
-us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to grade against $9,563.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Correctly took the $24,150 standard deduction plus the $700 non-itemizer charitable deduction and used the right 2026 brackets. However, it added the $68 state refund to income, even though the non-itemizing head got no prior-year tax benefit from it, and dropped the $8 IRA deduction. That $76.86 of excess taxable income at 22% accounts for its $16.91 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Treated the $33,000 long-term loss as wiping out ordinary income and reported $0 tax, ignoring the §1211(b) rule that net capital losses offset ordinary income only up to $3,000. With the capped loss, taxable income is $92,735.15 and tax after the $2,800 of credits is $9,563.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the $700 non-itemizer charitable deduction that applies from 2026. It also added the $68 refund to AGI ($117,662) and dropped the $8 IRA deduction, reaching taxable income of $93,512 instead of $92,735.15 and a $171 overstatement."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Netted the full $30,711 capital loss against income (AGI $90,132) instead of capping the deduction at $3,000. It used a $21,900 standard deduction and treated only $400 of the CTC as nonrefundable. It then submitted $12,866.90, which contradicts its own final computation of $6,509."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Filed single with a $15,750 standard deduction instead of head of household with $24,150, and claimed a $6,237 QBI deduction with no qualified business income. Its AGI of $139,775 is far above the correct $117,585.14. It also applied a $1,875 CTC instead of $2,200 and dropped the $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work ends at about $10,353, but it submitted $14,748, which none of its steps support. That work also left out the $700 non-itemizer charitable deduction and the $68 SALT refund, and it used a $2,000 CTC instead of the 2026 $2,200 and wrong HoH brackets (12% band ending at $65,900 instead of $67,450)."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The structure was right, but it computed $10,444.60 of ordinary tax on $79,380. The 2026 HoH brackets (10% to $17,700, 12% to $67,450, 22% above) give $10,364.60, so its figure is $80 too high. It also deducted the $8 traditional IRA contribution, which is fully phased out for an active 401(k) participant at about $117.6k AGI."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It used the single-filer $14,600 standard deduction instead of the $24,150 HoH amount. It also netted the entire $33,000 long-term loss against income instead of capping the net capital loss deduction at $3,000. Its final $23,437 then contradicts its own roughly $8,506 pre-credit tax with no derivation."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rounded guess that mentions only the CDCC and never subtracts the $2,200 nonrefundable CTC. The $14,500 is above even the correct pre-credit tax of $12,379.77, so it also overstated regular tax: it never applied the $3,000 capital-loss limit, the 15% qualified-dividend rate, or the $24,850 of standard plus charitable deductions."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $23,500 HoH standard deduction instead of $24,150 and pre-OBBBA brackets (10% to $17,000, 12% to $64,850) instead of $17,700/$67,450. It also left out the $700 non-itemizer charitable deduction and the $68 SALT refund income, so it overstated ordinary tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $23,000 HoH standard deduction, 2025-style brackets (12% band ending at $64,850), and a $2,000 CTC instead of the 2026 $2,200, and it left out the $700 non-itemizer charitable deduction. That gave $10,460, which it then raised to $11,150 without any basis."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It guessed a $24,500 standard deduction instead of $24,150, left out the $700 non-itemizer charitable deduction, and taxed ordinary income with 10% to $17,000 and 12% to $64,850 instead of the 2026 HoH thresholds of $17,700 and $67,450. That inflated its ordinary tax to $10,716 versus about $10,366."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $8 traditional IRA contribution. The head is an active participant in an employer 401(k), and AGI of about $117.6k is above the HoH deduction phase-out range, so the IRA deduction is zero. That $8 at 22% is the whole shortfall."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It counted the $2,289 short-term gain as separate ordinary income even though it nets into the $30,711 capital loss. It left the $13,424 of qualified dividends out of AGI and then subtracted them again from taxable income, so they were excluded twice. It also used a $22,500 standard deduction, a $2,000 CTC, and pre-OBBBA brackets."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own steps produced $10,739, but it submitted $13,920 with no supporting computation. Those steps also used a $22,500 standard deduction, 2024-level brackets (10% to $16,550, 12% to $63,100), a $2,000 CTC, and no $700 non-itemizer charitable deduction."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $24,150 standard deduction, the brackets, and both credits right. It left out the $700 charitable deduction that non-itemizers can take starting in 2026 (up to $1,000 for HoH), so taxable income was $700 too high and tax was overstated by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted both the full $33,000 long-term loss and a separate $3,000 capital-loss deduction instead of capping the net loss deduction at $3,000. It invented a $46,000 HoH standard deduction. It also applied a 35% CDCC rate on a $5,000 expense cap (correct is 20% of $3,000) and a $300 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: $9,680 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC. The One Big Beautiful Bill Act made the TCJA structure permanent, so 2026 has no personal exemptions, a $24,150 HoH standard deduction, a $2,200 CTC, and 10/12/22% brackets."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired: $11,000 of personal exemptions, pre-2018 brackets, and a $1,000 CTC phased out above the old $75,000 HoH threshold. Under OBBBA in 2026 there are no exemptions, the HoH standard deduction is $24,150, and the $2,200 CTC phases out only above $200,000."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $23,100 HoH standard deduction instead of $24,150, a $2,300 CTC instead of $2,200, and brackets that taxed $81,130 of ordinary income at $11,059.60 instead of about $10,750 under the 2026 thresholds. It also deducted the $8 IRA contribution, which is phased out, and left out the $700 non-itemizer charitable deduction."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA law: uncapped SALT itemizing of $18,564, $10,000 of personal exemptions, and a $1,000 CTC. Under 2026 OBBBA law there are no personal exemptions, the CTC is $2,200, and the $24,150 standard deduction plus the $700 non-itemizer charitable deduction beats itemizing."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $100,571 AGI is about $17,090 below the correct $117,661 because it did not include all $20,480 of dividends and limit the capital loss to $3,000. It also never subtracted the $600 CDCC and used a $2,000 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $21,208 of employer-sponsored insurance premiums from wages, giving $96,446 AGI, and applied pre-TCJA personal exemptions of $10,400 with no CTC. In 2026 there are no personal exemptions, and the $2,200 CTC applies in full because AGI is far below the $200,000 phase-out."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reduced AGI to $96,454 by subtracting the employer-sponsored insurance premiums, which are not part of the input wages. It also applied TCJA-sunset personal exemptions and a $1,000 CTC instead of the 2026 OBBBA rules: no exemptions, the $24,150 standard deduction plus $700 charitable deduction, and a $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. $12,790.25 is roughly the correct pre-credit tax of $12,379.77, which fits a calculation that never subtracted the $2,800 of nonrefundable credits (the $2,200 CTC and $600 CDCC) that bring the tax to $9,579.77."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It explicitly subtracted personal exemptions. Those are permanently zero under OBBBA, so its taxable income was too low and its tax understated by about $2,200. The 2026 HoH return gets only the $24,150 standard deduction and the $700 non-itemizer charitable deduction."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,384 fits netting the entire $30,711 capital loss against income. That drops AGI to about $90k and leaves the qualified dividends in the 0% bracket. The capital loss deduction is capped at $3,000, so AGI is $117,661 and all $13,424 of dividends are taxed at 15%."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It subtracted a $2,000 CTC instead of the 2026 $2,200 and left out the $700 non-itemizer charitable deduction. It also overstated pre-credit tax by about $320 beyond those two items by using outdated HoH bracket or standard deduction amounts, ending $675 high."
+us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the HoH standard deduction at $23,063 instead of $24,150 and the brackets at 10% to $16,913 and 12% to $64,575 instead of $17,700 and $67,450. It left out the $700 non-itemizer charitable deduction and the SALT refund income, and it used a $2,000 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 of employer-sponsored insurance premiums as deductible medical expenses and itemized $23,266. Without the premiums, medical costs fall below the 7.5% floor, and even its itemized figure is below the $24,850 available to a non-itemizer ($24,150 standard plus $700 charitable). It also overstated ordinary tax with wrong bracket thresholds."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated capital gains as positive taxable income instead of recognizing a $30,711 net capital loss limited to a $3,000 deduction. Its unsupported $12,719 is $3,139 above the correct figure, more than the $2,800 of CTC and CDCC it says it subtracted."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized real estate taxes, medical, and charity, but medical costs are below the 7.5% AGI floor and the itemized total is below the $24,850 of standard plus charitable deductions available to a non-itemizer. Its $17,303 is above even the correct pre-credit tax of $12,379.77, so it also skipped the $3,000 capital-loss limit, the 15% qualified-dividend rate, or the $2,800 of nonrefundable credits."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $24,415, which is less than the $24,850 a non-itemizer gets ($24,150 HoH standard deduction plus the $700 charitable deduction for non-itemizers). It also subtracted a $2,000 CTC instead of the 2026 $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its AGI of about $89,951 nets the entire $30,711 capital loss against income. The net capital loss deduction is capped at $3,000, so AGI is $117,661, and the qualified dividends are taxed at 15% rather than falling into the 0% bracket."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It got AGI, the standard deduction, the brackets, and both credits right. It left out the 2026 non-itemizer charitable deduction ($700 of cash gifts, up to $1,000 for HoH), so taxable income was $700 too high and tax was overstated by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the right $2,200 CTC and $600 CDCC but left out the $700 non-itemizer charitable deduction. It also overstated pre-credit tax by about $500 more through understated HoH bracket thresholds or standard deduction, implying about $13,036 of pre-credit tax versus $12,379.77."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses, which only exceed the 7.5% AGI floor if the $21,208 of employer-sponsored insurance premiums are counted, and those premiums are not deductible. Real estate taxes, NY income tax, and charity alone come to less than the $24,850 non-itemizer total, so its deduction was too large and its tax understated."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the $24,150 standard deduction, the 2026 brackets, and the $2,200 CTC and $600 CDCC correctly. It left out the $700 charitable deduction for non-itemizers that applies starting in 2026, overstating tax by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $93,504 taxable income leaves out the $700 non-itemizer charitable deduction. It also includes an $8 traditional IRA deduction that is fully phased out for an active 401(k) participant at this AGI. The missing $700 deduction accounts for the $152 overstatement after the $8 offset."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses, which requires counting the non-deductible $21,208 of employer-sponsored insurance premiums. That pushed its itemized total above the correct deduction of $24,850 ($24,150 standard plus the $700 non-itemizer charitable deduction) and understated tax by about $778."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. $14,000 is above even the correct pre-credit tax of $12,379.77, so it never subtracted the $2,200 CTC and $600 CDCC and also overstated regular tax by ignoring the $3,000 capital-loss deduction and the $24,850 of standard plus charitable deductions."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026: two personal exemptions and a CTC fully phased out. OBBBA made the TCJA structure permanent, so there are no exemptions and the $2,200 CTC applies in full. It also wrongly subtracted the $21,208 of employer-sponsored insurance premiums from wages, giving $96,454 AGI."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: two $5,300 exemptions, 10/15/25% HoH brackets, and a $1,000 CTC phased out above $75,000. In 2026 under OBBBA there are no exemptions, brackets are 10/12/22%, the $24,150 standard deduction plus $700 charitable deduction applies, and the $2,200 CTC phases out only above $200,000."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its $28,094 of itemized deductions includes medical expenses above the 7.5% floor, which only works by counting the non-deductible $21,208 of employer-sponsored insurance premiums. Without them, itemizing falls below the $24,850 of standard plus non-itemizer charitable deductions, so it understated taxable income."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules: a $12,591 HoH standard deduction, pre-2018 brackets, and no CTC. Under 2026 OBBBA law the HoH standard deduction is $24,150, the $700 non-itemizer charitable deduction applies, and the $2,200 CTC is subtracted in full."
+us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $700 non-itemizer charitable deduction and deducted the phased-out $8 IRA contribution, giving $93,504 of taxable income instead of $92,811. It then taxed $80,080 of ordinary income at about $10,638 instead of $10,519 under the 2026 HoH brackets (12% to $67,450)."
+us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the capital loss as wiping out the tax. The net capital loss deduction is capped at $3,000, leaving $92,811 of taxable income and $12,379.77 of pre-credit tax, which the $2,800 of nonrefundable credits cannot eliminate."
+us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI, $24,150 standard deduction, brackets, and both credits are right. It left out the $700 charitable deduction for non-itemizers that applies starting in 2026, so taxable income was $93,512 instead of $92,811 and tax was overstated by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full $30,711 capital loss against income instead of capping the deduction at $3,000. It used a $21,900 standard deduction and treated only $400 of the CTC as nonrefundable, reaching $6,509, and then submitted an unsupported $12,866.90."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It used the single-filer $15,750 standard deduction instead of the $24,150 head-of-household amount. It invented a $6,237 QBI deduction even though there is no business income, and it omitted the $600 CDCC while using a $1,875 CTC instead of $2,200."
us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied only the ACTC limitation formula — lesser of $1,700 and 15% of earned income over $2,500 — and never performed the ordering step in which the nonrefundable CTC is first applied against tax liability and only the unused remainder becomes refundable. Its own reasoning conceded the head 'has sufficient nonrefundable CTC capacity' and then discarded that conclusion; with roughly $117,650 of AGI the pre-credit federal tax absorbs the entire credit, leaving $0. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200 amount."
us,scenario_082,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the $1,700 statutory refundable ceiling as the answer after checking only the $2,500 earned-income floor, skipping the requirement that the refundable ACTC equals the CTC remaining after the nonrefundable portion offsets tax liability. Even at its own stated AGI of about $86,949, head-of-household tax before credits far exceeds the $2,200 CTC, so the credit is fully used nonrefundably and refundable credits are $0."
us,scenario_082,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It quoted $1,000, the pre-TCJA additional child tax credit cap, which has no basis in 2026 law where the credit is $2,200 per child with a $1,700 refundable ceiling. The controlling error is treating a refundable amount as automatic: this filer's federal tax before credits exceeds the full $2,200 credit, so the nonrefundable CTC absorbs all of it and $0 is refundable."
@@ -6751,146 +6925,151 @@ us,scenario_082,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,
us,scenario_082,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It used $1,600, the 2023 refundable ceiling rather than the 2026 $1,700 cap on a $2,200 credit, and then treated that cap as an automatic entitlement once earned income cleared $2,500. The refundable portion arises only to the extent the credit exceeds tax liability; this filer's federal tax before credits absorbs the full $2,200, so refundable credits are $0."
us,scenario_082,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It reported $1,875, which matches no statutory ACTC parameter — the 2026 refundable ceiling is $1,700 against a $2,200 credit — and treated the refundable portion as automatic given sufficient earned income. Federal tax before credits on roughly $117,650 of AGI exceeds the entire credit, so the CTC is taken wholly as a nonrefundable credit and $0 is refundable."
us,scenario_082,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_082,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model imposed NYC resident income tax solely because the household lives in New York State. It should not have applied NYC rates without a listed NYC location; the separate local-income-tax liability is zero.
-us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model estimated NYC personal income tax from adjusted gross income even though the prompt specifies only New York State and treats unlisted location facts as false. With no NYC residence or other taxable locality listed, local income tax is zero."
-us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model calculated NYC head-of-household tax brackets on $79,254 of taxable income without any fact establishing NYC residence. New York State residence alone triggers no local income tax in this benchmark, yielding zero."
-us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"The model explicitly assumed NYC residence because NYC is a New York locality with an income tax, contradicting the instruction that unlisted household facts are false. Its detailed NYC taxable-income, bracket, school-credit, and dependent-care-credit computation is inapplicable because no NYC location was provided."
-us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"The model treated the household as an NYC resident and applied NYC brackets and credit rules despite the prompt listing only New York State. Without a specified NYC residence, none of those local rules applies and the local-income-tax output is zero."
-us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It computed federal FICA at $7,664.92 and left out New York's PFL and disability contributions, calling them negligible and excluded. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It summed only employee Social Security ($6,212.09) and Medicare ($1,452.83), $7,664.92, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It applied a 0.62% New York PFL rate to the full $100,195 with no cap ($621.21), then added an unspecified amount for other New York withholdings, for $640 of state contributions. The reference counts the most the employer may collect, $411.91 of PFL (0.432% of wages up to the cap) and $31.20 of DBL, or $443.11, so the model's state part is about $197 over it."
-us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It counted DBL at $31.20 but dropped its own PFL estimate (0.388%, capped at about $354) for an unsupported $49.31, so it added only $80.51 of New York contributions. The reference counts the most the employer may collect: PFL at the 2026 rate of 0.432% up to the $411.91 cap, plus DBL of $31.20, or $443.11, for $8,108.03."
-us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It said New York has no mandatory employee payroll tax and reported only the $7,664.92 of federal FICA. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It said New York imposes no mandatory employee payroll tax counted here and reported rounded FICA, $7,665. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It stopped at federal FICA, $7,664.92, on the assumption that no mandatory New York employee payroll tax is modeled. The engine does include New York's contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It counted DBL at $31.20 but computed PFL at 0.388% on a cap of about $89,343, $346.65, for $8,042.77. The reference counts the most the employer may collect at 2026 parameters, PFL of 0.432% of wages up to $411.91, so the total is $65.26 under it."
-us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said New York's disability and PFL employee contributions are minimal and added an unsupported $129.39 estimate, for $7,794.31. The reference counts the most the employer may collect, $411.91 of PFL (0.432% of wages up to the cap) and $31.20 of DBL, or $443.11, so the model's state part is about $314 under it."
-us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It said no New York payroll tax is modeled and reported only FICA, $7,664.92. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It subtracted the $181 traditional 401(k) deferral from FICA wages ($100,014) and said New York has no mandatory employee state payroll taxes, for $7,651. The engine applies Social Security and Medicare to the full $100,195. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It included New York's disability contribution ($31.20) but not PFL, for $7,696.12. The reference also counts PFL at the most the employer may collect, 0.432% of wages up to the $411.91 cap, so the total is $411.91 under it; the law lets the employer collect less or none of either."
-us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It used a New York PFL maximum of $369.68 instead of the 2026 maximum of $411.91 and counted no DBL, for $8,034.60. The reference counts the most the employer may collect, $411.91 of PFL and $31.20 of DBL, so the total is $73.43 under it; the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It used the 2026 PFL rate of 0.432% but applied it to all $100,195 of wages ($432.84) without the cap, which limits the most the employer may collect to $411.91. The total is therefore $20.93 over the reference."
-us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It included DBL ($31) but used a New York PFL figure of about $333, below the 2026 maximum of $411.91 (0.432% of wages up to that cap), so the total is about $79 under the reference."
-us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% federal FICA rate to wages, $7,665, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 employer-sponsored insurance premium from wages, giving a FICA base of $78,987. The payroll tax base is the full $100,195, so Social Security and Medicare come to $7,664.92. It also put PFL at $294.62, which is 0.373% of the reduced base, against the 2026 maximum of $411.91 that the reference counts."
-us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It reduced FICA wages to $78,987 by subtracting the $21,208 employer-sponsored insurance premium, which understates Social Security and Medicare by $1,622.42 ($6,042.50 against the engine's $7,664.92 on the full $100,195). It used about $394 for PFL and counted no DBL. The reference counts the most the employer may collect, $411.91 of PFL and $31.20 of DBL, for $8,108.03; the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It counted only employee Social Security and Medicare on gross wages, $7,664.92, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It computed FICA on $78,987 after subtracting the $21,208 employer-sponsored insurance premium, instead of the full $100,195, which understates federal payroll tax by about $1,622. Its New York disability and PFL add-on of about $270 is under the $443.11 the reference counts, the most the employer may collect ($411.91 of PFL and $31.20 of DBL); the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It treated the $21,208 employer-sponsored insurance premium as reducing FICA wages to $78,987, but the engine applies Social Security and Medicare to the full $100,195. Its New York disability and PFL amount of about $333 is under the $443.11 the reference counts, the most the employer may collect ($411.91 of PFL and $31.20 of DBL); the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It reported only employee Social Security ($6,212.09) and Medicare ($1,452.83), $7,664.92, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"It included DBL ($31.20) but computed PFL with the 2025 rate of 0.388% on a guessed $96,200 cap ($373.26). The reference counts the most the employer may collect at the 2026 rate of 0.432%, up to the $411.91 cap, so the total is $38.65 under it."
-us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It said New York has no employee-side state payroll tax and mis-added its own components: $6,212 + $1,453 is $7,665, not $7,765. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It said New York has no mandatory employee state payroll tax and reported FICA rounded down to $7,660 (the engine's is $7,664.92). The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It reported only federal Social Security and Medicare, under-rounded to $7,659 instead of $7,664.92, without mentioning New York's PFL or disability contributions. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Its New York add-on of $385.73 equals the 2025 PFL maximum ($354.53) plus DBL ($31.20). Against the 2026 PFL maximum of $411.91, the most the employer may collect, the total is $57.38 under the reference."
-us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It added New York's disability contribution ($31.20) but not PFL, for $7,696.12. The reference also counts PFL at the most the employer may collect, 0.432% of wages up to the $411.91 cap, so the total is $411.91 under it; the law lets the employer collect less or none of either."
-us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% federal FICA rate, $7,664.92, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It reported only Social Security and Medicare and understated even that ($7,654 instead of $7,664.92), without mentioning New York's PFL or disability contributions. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It applied the 2026 PFL rate of 0.432% to all $100,195 of wages ($432.84) without the cap, which limits the most the employer may collect to $411.91. The total is therefore $20.93 over the reference."
-us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% federal FICA rate, $7,665, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It subtracted the $21,208 employer-sponsored insurance premium from wages, giving a FICA base of $78,987, when the engine applies Social Security and Medicare to the full $100,195, and it said New York has no employee payroll tax. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It counted New York PFL at about $367 instead of the 2026 maximum of $411.91 and said no other employee state payroll tax applies, so it counted no DBL, for $8,032. The reference counts the most the employer may collect, $411.91 of PFL and $31.20 of DBL, so the total is about $76 under it; the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"It reported only employee Social Security ($6,212.09) and Medicare ($1,452.83), rounded to $7,665, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It subtracted both the $21,208 employer-sponsored insurance premium and the $181 traditional 401(k) deferral from FICA wages ($78,806), when the engine applies Social Security and Medicare to the full $100,195, and it did not mention New York's PFL or disability contributions. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It included DBL (about $31) but estimated PFL at about $365, below the 2026 maximum of $411.91 (0.432% of wages up to that cap), so the total is about $47 under the reference."
-us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there is no answer to score. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
-us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It computed federal FICA, $7,664.92, rounded to $7,665, and stopped there, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It summed only employee Social Security and Medicare, $7,665, without mentioning New York's PFL or disability contributions. The reference adds $443.11, the most the employer may collect: Paid Family Leave at 0.432% of wages up to its $411.91 cap, and disability benefits at $0.60 a week ($31.20). The law lets the employer collect less or none, and the prompt does not say what it does; counting only what the law requires of the employee gives federal tax alone, $7,664.92."
-us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It found FICA of $7,664.92 and at one point estimated New York PFL at about $431.88, then subtracted an invented '~$-200' of New York deductions and reported $7,464.33, below FICA alone. The reference adds New York's contributions rather than subtracting them: $411.91 of PFL and $31.20 of DBL, the most the employer may collect, for $8,108.03; the law lets the employer collect less or none."
-us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It said it included no mandatory employee state payroll tax and mis-added its own components: $6,212.09 + $1,452.83 is $7,664.92, not $7,752.42. The reference is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 Paid Family Leave (PFL) + $31.20 disability benefits (DBL) = $8,108.03, counting the most the employer may collect for each (PFL at 0.432% of wages up to the $411.91 cap, DBL at $0.60 a week); the law lets the employer collect less or none, and without them the total is $7,664.92."
+us,scenario_082,local_income_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It estimated 'NYC resident income tax' of $3,700, but nothing in the household facts says the household lives in NYC. Under the rule that unlisted status inputs are false, the household is a non-NYC New York resident, which leaves no local income tax and gives $0."
+us,scenario_082,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied NYC graduated rates (3.078%–3.819%) to $107,086 of state taxable income, which assumes NYC residence even though the household is listed only as NY state. With NYC residence unlisted and therefore false, no NYC tax applies and local income tax is $0."
+us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed an NYC personal income tax of $3,150 from AGI, assuming without basis that the household lives in New York City. State = NY alone does not trigger NYC tax, and unlisted NYC residence defaults to false, so the correct local income tax is $0."
+us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied NYC head-of-household brackets to an NYC taxable income of $79,254, treating the household as NYC residents when only state = NY was given. Because NYC residence is an unlisted status that defaults to false, no NYC tax arises and local income tax is $0."
+us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"It said outright that it 'assumed to reside in NYC (the only NY locality with a local income tax),' which breaks the instruction to treat unlisted facts as false. Without NYC residence, its whole NYC bracket, school tax credit and child care credit computation does not apply, and local income tax is $0."
+us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"It stated that 'NYC resident income tax applies' to NY taxable income of about $104,454 and applied NYC brackets, but the household is never placed in NYC. Under the default-false rule for unlisted status facts, the household owes no NYC tax and local income tax is $0."
+us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 but deliberately left out NY PFL ($411.91) and DBL ($31.20). It read 'mandatory employee state payroll taxes' as excluding contributions that NY employers are allowed but not required to withhold. The reference counts the statutory maximum employee contributions, which add $443.11."
+us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($6,212.09 + $1,452.83 = $7,664.92). It left out the NY employee PFL contribution (0.432% of wages, capped at $411.91) and the DBL contribution ($31.20), which the reference counts at their maximums as mandatory employee state payroll taxes."
+us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Applied a made-up 0.62% PFL rate with no cap ($621.21), then padded the total with unnamed 'other NY withholdings' to reach $8,304.92. In 2026 the NY PFL employee rate is 0.432% with a $411.91 annual maximum, and DBL is $31.20, so NY adds $443.11."
+us,scenario_082,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665). It left out NY PFL ($411.91, 0.432% capped) and DBL ($31.20), reading the employer-optional NY employee contributions as not mandatory, while the reference includes them at the statutory maximum."
+us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Got FICA and DBL ($31.20) right, but after citing the 2025 0.388% PFL rate it fell back to an unexplained $49.31 PFL figure. The 2026 PFL is 0.432% of wages capped at $411.91, so NY adds $443.11, not $80.51."
+us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and reported FICA only ($7,664.92). That left out the NY PFL ($411.91) and DBL ($31.20) employee contributions, which the reference treats as mandatory employee state payroll taxes at their statutory maximums."
+us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665) and said NY imposes no counted employee payroll tax. The reference adds NY PFL at 0.432% capped at $411.91, plus DBL at $31.20."
+us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 and treated NY employee payroll contributions as not modeled. The reference includes NY PFL ($411.91, at the 2026 cap) and DBL ($31.20) as mandatory employee state payroll taxes."
+us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Included both NY items but used the outdated 0.388% PFL rate on an ~$89,343 wage cap ($346.65). The 2026 PFL is 0.432% of wages with a $411.91 maximum, which binds at $100,195, so PFL came out $65.26 short."
+us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92, then added an arbitrary $129.39 'SDI/PFL estimate'. The correct NY add-on is PFL $411.91 (0.432% capped) plus DBL $31.20, or $443.11."
+us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92) on the view that no NY payroll tax applies. It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at the statutory maximum employee contribution."
+us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Subtracted the $181 traditional 401(k) deferral from FICA wages, giving $100,014. Elective deferrals are still Social Security and Medicare wages under IRC §3121(v)(1)(A), so the base is $100,195. It also left out NY PFL ($411.91) and DBL ($31.20)."
+us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Added NY DBL ($31.20) but left out the larger NY PFL employee contribution: 0.432% of $100,195, capped at the 2026 maximum of $411.91. This undercounted NY payroll tax by $411.91."
+us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,Used an outdated PFL maximum of $369.68 instead of the 2026 maximum of $411.91 (0.432% rate). It also left out the $31.20 DBL contribution.
+us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Applied the correct 0.432% PFL rate but skipped the 2026 annual maximum, reporting $432.84 instead of the capped $411.91. That overstated payroll tax by $20.93."
+us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Used about $333 for NY PFL, which is the older 2024 maximum, instead of the 2026 maximum of $411.91 (0.432% of wages, capped). It also rounded DBL to $31."
+us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Reported only 7.65% FICA on $100,195 ($7,665). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts as mandatory state payroll taxes."
+us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premium input from FICA wages, giving $78,987. The listed premium is not a salary reduction of the $100,195 gross wages, so Social Security and Medicare apply to the full $100,195. It also figured PFL at an outdated 0.373% on the reduced base ($294.62) instead of the $411.91 cap."
+us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Cut FICA wages to $78,987 by subtracting the ESI premium input. Employee Social Security and Medicare apply to the full $100,195 gross wages. It also used about $394 for PFL instead of $411.91 and left out DBL ($31.20)."
+us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92). It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at the statutory maximum as mandatory employee state payroll taxes."
+us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Computed FICA on $78,987 after subtracting the $21,208 ESI premiums. The FICA base is the full $100,195 ($7,664.92 FICA), and NY PFL/DBL add $443.11."
+us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Treated $78,987 (gross wages minus ESI premiums) as FICA-taxable wages. Employee Social Security and Medicare apply to the full $100,195, and NY adds PFL $411.91 plus DBL $31.20."
+us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Computed Social Security plus Medicare only ($7,664.92) and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts."
+us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Used the outdated 0.388% PFL rate on an assumed $96,200 wage cap ($373.26). The 2026 PFL rate is 0.432% with a $411.91 annual maximum, so PFL came out $38.65 short. DBL ($31.20) was correct."
+us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Listed components of $6,212 + $1,453 = $7,665 but reported $7,765, a $100 addition error. It also left out NY PFL ($411.91) and DBL ($31.20)."
+us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and reported a rounded FICA-only $7,660, slightly below the exact $7,664.92. It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at their statutory maximums."
+us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Reported FICA only, rounded down to $7,659 (exact is $7,664.92). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference includes."
+us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Estimated combined NY PFL/DBL at $385.73. The 2026 PFL is 0.432% of wages capped at $411.91, plus DBL of $31.20, for $443.11, so it used an outdated PFL rate or cap."
+us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Included NY DBL ($31.20) but left out NY PFL, a 0.432% employee contribution that reaches the $411.91 annual maximum at $100,195 of wages."
+us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Reported 7.65% FICA only ($7,664.92). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference treats as mandatory state payroll taxes."
+us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Reported FICA only at $7,654, which is also $10.92 below the exact 7.65% × $100,195 = $7,664.92. It left out NY PFL ($411.91) and DBL ($31.20), which the reference includes."
+us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Applied the 0.432% PFL rate without the 2026 annual maximum, reporting $432.84 instead of the capped $411.91. That overstated the total by $20.93."
+us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Reported 7.65% FICA only ($7,665) and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts."
+us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premiums from FICA wages ($78,987). Employee Social Security and Medicare apply to the full $100,195. It also explicitly said NY has no employee payroll tax, leaving out PFL ($411.91) and DBL ($31.20)."
+us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Used about $367 for NY PFL instead of the 2026 maximum of $411.91 (0.432% capped). It also said no other NY employee payroll tax applies, which left out the $31.20 DBL contribution."
+us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,665) and left out NY PFL ($411.91) and DBL ($31.20), which the reference includes at their statutory maximums."
+us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Subtracted both the ESI premiums ($21,208) and the 401(k) deferral ($181) from FICA wages, giving $78,806. Elective 401(k) deferrals are FICA wages, and the full $100,195 is the payroll-tax base. It also left out NY PFL/DBL ($443.11)."
+us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Estimated NY PFL at about $365 instead of the 2026 maximum of $411.91 (0.432% of wages, capped). DBL ($31) was about right."
+us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no payroll_tax value or explanation, so there was no answer to score."
+us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Computed FICA ($7,664.92), rounded it to $7,665, and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts."
+us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,665) and never added the NY PFL ($411.91 at the 2026 cap) or DBL ($31.20) employee contributions that the reference includes."
+us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Tried several NY estimates and then settled on an unexplained negative ~$200 'NY adjustment', reporting $7,464.33, which is below FICA alone. NY PFL ($411.91) and DBL ($31.20) add $443.11 to the $7,664.92 FICA."
+us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Listed $6,212.09 + $1,452.83 + $0 Additional Medicare Tax but reported $7,752.42, an unexplained $87.50 extra. It also explicitly left out NY PFL ($411.91) and DBL ($31.20)."
us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It subtracted a ~$396 NY child and dependent care credit as if it were nonrefundable, but New York's CDCC is refundable and excluded from this output. It also never added the $156.38 supplemental recapture tax that applies because NY AGI is above $107,650. Its ~$5,891 bracket tax on $105,454 is too high: the 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) gives about $5,446."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its bracket tax was right (~$5,446), but it put the supplemental recapture tax at only $49.75 by measuring the benefit against the 5.4% rate of its own bracket. New York's recapture for a HOH filer with NY AGI above $107,650 comes to $156.38 here. It also skipped the $68 state-refund subtraction, which slightly overstates taxable income."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used federal taxable income ($62,149, after the $14,600 federal standard deduction) as NY taxable income. The correct path starts from NY AGI of $117,585 and subtracts the $11,200 NY HOH standard deduction and the $1,000 dependent exemption, giving $105,385.15. Its $5,649 is not even the NY tax on $62,149 (about $3,100 on the HOH schedule), and it leaves out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed the head as single despite a qualifying 1-year-old child. That meant the $8,000 single standard deduction instead of $11,200 HOH, no $1,000 dependent exemption, and single brackets. It also overstated AGI at ~$119,943 (federal AGI is $117,654). It then cut its $6,221 to $5,076 with an unexplained ~$1,145 reduction, although no nonrefundable NY credit applies."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It filed as single rather than head of household: the $8,000 single deduction, no $1,000 dependent exemption, and 6% on income above $80,650. It also skipped the $181 traditional 401(k) and $8 IRA reductions. It then took ~$1,050 off its $6,326 for a 'child/dependent care nonrefundable portion', but New York's CDCC is refundable and the household credit is zero at this income."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used head of household but took an ~$8,000 standard deduction (the single amount) instead of the $11,200 HOH deduction, and it left out the $1,000 dependent exemption. That gave taxable income of ~$109,654 instead of $105,385.15. It then rounded to $6,000 using 5.5%–6% marginal rates, instead of the 2026 HOH schedule (5.4% in this bracket) plus the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It got taxable income right ($105,386) but put the 2026 HOH bracket tax at $5,808.15. The schedule gives $5,442.17 (12,800×3.9% + 4,850×4.4% + 3,250×5.15% + 84,485×5.4%). It also understated the supplemental recapture tax at $81.40 versus $156.38."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated the head as single, so it compared $9,917 of itemized deductions with the $8,000 single deduction instead of $11,200 HOH. It dropped the $1,000 dependent exemption and used made-up single brackets (5.85% above $27,900). It also subtracted a $120 NY child and dependent care credit, which is refundable and excluded here, and never added the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It filed as single and itemized $9,917 against the $8,000 single deduction, missing the $11,200 HOH standard deduction and the $1,000 dependent exemption. It then used single brackets (6.25% above $80,650) to get $6,193. It cut that to $4,800 for the household credit, which is fully phased out at this income; no nonrefundable NY credit applies."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Its taxable income was right (~$105,394), but it rounded the graduated tax to $5,400. It never added the $156.38 supplemental recapture tax, which applies because NY AGI of $117,585 exceeds $107,650. The 2026 HOH schedule gives $5,442.17, so the total is $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put federal AGI at ~$86,949, which matches deducting the net capital loss far beyond the $3,000 annual cap; NY AGI is $117,585. It also used a ~$10,000 standard deduction instead of $11,200 and claimed $2,000 of exemptions when only the child gets the $1,000 exemption. It subtracted the Empire State Child Credit, which is refundable and excluded from this output."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of NY itemized deductions. The NY itemized total here is only $11,169 (SALT $11,057 plus a reduced $112 charitable deduction), below the $11,200 HOH standard deduction, so taxable income is $105,385.15, not $95,971. It also used pre-2026 rates (5.5% instead of 5.4%) with wrong bracket widths and left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It left out the $1,000 dependent exemption for the child, so taxable income was $106,394 instead of $105,385.15. Its $5,731.77 pre-credit tax does not match the 2026 HOH schedule plus supplemental tax. It then subtracted a $120 NY child and dependent care credit, which is refundable in New York and excluded from tax before refundable credits."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted $2,000 of exemptions, but New York gives the filer no exemption, only $1,000 per dependent, so taxable income is $105,385.15, not $104,386. It also left out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 HOH amount and left out the $1,000 dependent exemption, giving taxable income of $107,086 instead of $105,385.15. It then subtracted a $30 household credit that is not available at this income. It never added the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its only explanation was 'wage income after standard deduction'. The $4,380 matches taxing about $84,150 at pre-2026 HOH rates, which is what you get by subtracting the $21,208 employer health premiums from the listed wages; those wages count in full in AGI. The correct path taxes $105,385.15 at 2026 HOH rates ($5,442.17) and adds the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $84,178 taxable income equals NY AGI minus the $21,208 employer-sponsored insurance premiums minus $12,200 of deduction and exemption. It wrongly took the premiums out of the listed gross wages; taxable income is $105,385.15. It also used pre-2026 HOH rates (4%/4.5%/5.25%/5.5%). Pushing AGI below $107,650 made it miss the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut NY AGI to $96,454 by subtracting the $21,208 employer-sponsored insurance premiums, but the listed wages are fully taxable and NY AGI is $117,585. It then subtracted $17,200 instead of the $11,200 HOH standard deduction plus the $1,000 dependent exemption. With AGI below $107,650 in its calculation, it also missed the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $6,040.50 is $442 more than the correct $5,442.17 HOH bracket tax plus $156.38 supplemental tax. That fits single-filer treatment ($8,000 deduction, no $1,000 dependent exemption, 6% above $80,650), which gives about $6,000, instead of head of household with $11,200 and the 2026 HOH schedule."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $4,220 corresponds to about $82,000–$83,000 of taxable income on the HOH schedule, roughly $23,000 below the correct $105,385.15, which matches taking the $21,208 employer health premiums out of income. It also left out the $156.38 supplemental tax that applies because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $3,591 corresponds to about $71,000 of taxable income on the HOH schedule, roughly $34,000 short of the correct $105,385.15. So it left out income that belongs in NY AGI: wages reduced only by the $181 traditional 401(k), $20,480 of dividends, and a capital loss capped at $3,000. It also left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct $11,200 HOH standard deduction and $1,000 dependent exemption. But its $5,533 follows the pre-2026 rates (4%/4.5%/5.25%/5.5%) instead of the 2026 cut rates (3.9%/4.4%/5.15%/5.4%). It never added the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so nothing could be scored against the $5,598.55 that comes from $105,385.15 of taxable income on the 2026 HOH schedule plus the supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It itemized $27,966 by adding ~$4,700 of NY income tax to SALT (New York does not allow deducting its own income tax) and a $13,349 medical deduction built on the employer-sponsored insurance premiums. NY itemized deductions here are only $11,169, below the $11,200 HOH standard deduction. It also subtracted a refundable $120 NY CDCC and left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave a vague estimate citing 'limited credits', but no nonrefundable NY credit applies: the CDCC and Empire State Child Credit are refundable and the household credit is phased out. The correct result also adds a $156.38 supplemental tax on top of the $5,442.17 HOH bracket tax. Its $5,150 is below even the bracket tax alone, which matches subtracting out-of-scope credits and skipping the supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It gave no computation and claimed NY itemized deductions from real estate taxes, medical, and charity. But NY itemized deductions ($11,169) are below the $11,200 HOH standard deduction, and the medical expenses fall under the floor. The correct $105,385.15 at 2026 HOH rates gives $5,442.17 plus $156.38. Its $7,777 overstates that by $2,178, an effective rate far above the 5.4% top rate that applies here."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got taxable income right ($105,394) but applied pre-2026 HOH rates (5.5% above $20,900), producing $5,548. The 2026 cut rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17. It also left out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $2,954.80 corresponds to only about $59,000 of taxable income on the HOH schedule, roughly $46,000 short of the correct $105,385.15. That gap means it dropped income that stays in NY AGI: the $21,208 employer premiums are not subtracted from the listed wages, and capital losses are capped at $3,000. It also left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $106,394 taxable income leaves out the $1,000 dependent exemption for the child. Its $5,815 is also more than the 2026 HOH schedule produces. Correct taxable income of $105,385.15 gives $5,442.17 at 3.9%/4.4%/5.15%/5.4%, plus the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted only the $11,200 standard deduction and never took the $1,000 dependent exemption for the child. Its $5,977.08 overstates the 2026 HOH tax. The correct $105,385.15 of taxable income gives $5,442.17 at 3.9%/4.4%/5.15%/5.4%, plus a $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized about $23,624, which requires a medical deduction built on the $21,208 employer-sponsored insurance premiums. NY itemized deductions here are only $11,169 (SALT $11,057 plus $112 charitable), so the $11,200 HOH standard deduction applies and taxable income is $105,385.15, not $92,961.65. The supplemental tax it added was computed on that understated base."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied the 2026 HOH brackets after the standard deduction and dependent exemption and landed near the $5,442.17 bracket tax. It left out the $156.38 supplemental recapture tax that New York imposes because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted a $120 NY child and dependent care credit as nonrefundable, but New York's CDCC is refundable and excluded from tax before refundable credits. Its $5,665.24 pre-credit tax is also above the correct $5,598.55 ($5,442.17 on the 2026 HOH schedule plus the $156.38 supplemental tax)."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized about $23,624, which requires a medical deduction built on the $21,208 employer-sponsored insurance premiums. NY itemized deductions here total only $11,169 (SALT $11,057 plus $112 charitable), so the $11,200 HOH standard deduction applies and taxable income is $105,385.15, not $92,961.65. Its $4,754.16 is the tax on that understated base."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It gave no derivation. Its round $5,500 sits just above the $5,442.17 2026 HOH bracket tax on $105,385.15, which matches leaving out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It set NY AGI to $96,454 by subtracting the $21,208 employer-sponsored insurance premiums from the listed wages, and it used a ~$12,100 HOH standard deduction instead of $11,200. It subtracted a $120 NY CDCC, which is refundable in New York. The understated AGI also meant it skipped the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed $901 + 5.5% over $20,900 on the correct $105,394, which is the pre-2026 HOH schedule. The 2026 cut rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17. It never added the $156.38 supplemental recapture tax."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It took about $23,000 of NY itemized deductions, but they total only $11,169 here, below the $11,200 HOH standard deduction, so taxable income is $105,385.15 rather than $93,545. It also used the single schedule ($4,271 + 6% over $80,650) instead of the HOH schedule. It left out the supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It claimed $2,000 of exemptions, but New York gives the $1,000 exemption only for dependents, not the filer. It overstated the bracket tax at ~$5,699 and subtracted a $120 NY CDCC that is refundable in New York. It also left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so nothing could be scored against the $5,598.55 that comes from $105,385.15 of taxable income on the 2026 HOH schedule plus the supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its taxable income of $105,394 is right, but it applied a HOH schedule running 'through the 5.75% bracket'. For 2026 the rate on HOH income from $20,900 to $107,650 is 5.4%, giving $5,442.17. It also left out the $156.38 supplemental recapture tax."
-us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It let the $33,000 long-term capital loss offset most income and treated the CTC as cutting NY tax to $0. But a net capital loss offsets only $3,000 of ordinary income, so NY AGI is $117,585 and taxable income is $105,385.15, taxed at 2026 HOH rates for $5,442.17 plus $156.38. It also used the $8,000 single deduction instead of $11,200 HOH."
-us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used an $8,000 standard deduction (the single amount) instead of the $11,200 HOH deduction. It applied outdated rates (5.85% up to $107,650, 6.25% above) instead of the 2026 HOH rates (3.9%/4.4%/5.15%/5.4%). It subtracted a $120 NY CDCC, which is refundable in New York, and left out the $156.38 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $30,711 net capital loss instead of capping it at $3,000, cutting federal AGI to $89,951 from $117,654. It added the $68 refund instead of subtracting it, used the $8,000 single deduction instead of $11,200 HOH, and applied single brackets. It then raised its $4,292.89 tax to $4,755.44 'after household credit', a credit that is zero at this income."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It overstated federal AGI at $139,586; with the capital loss capped at $3,000 it is $117,654. It used the $8,000 single deduction instead of the $11,200 HOH deduction and subtracted an invented 'federal deduction adjustment'. It then subtracted the refundable Empire State Child Credit and a household credit that is phased out at this income."
-us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the old ESCC formula of 33% of a $1,000 pre-TCJA federal CTC, giving $330, instead of the enhanced $1,000 under-4 credit less the $693 phase-out, which gives $307. It also guessed about $150 for the NY CDCC instead of 60% of the $600 federal credit, which is $360."
-us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Applied a 20% NY CDCC percentage and got $120 instead of the 60% percentage, which gives $360; this $240 gap accounts for most of the miss. It also applied the ESCC phase-out as a continuous 1.65% ($703.79) instead of $16.50 per whole $1,000 over $75,000 (42 steps, $693), so it got $296.21 instead of $307."
-us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that New York has no refundable child credit, missing the refundable Empire State Child Credit ($307 after phase-out) and the refundable NY child and dependent care credit ($360)."
-us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Invented a $110 under-4 ESCC instead of applying the $1,000 under-4 amount reduced by $693 to $307. It also wrongly treated the NY child and dependent care credit as nonrefundable at this AGI, dropping the refundable $360 (60% of the $600 federal CDCC)."
-us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Assumed the ESCC phases out completely at this income. The $16.50-per-$1,000 reduction above $75,000 removes only $693 of the $1,000 under-4 credit, leaving $307. It also wrongly called the NY CDCC nonrefundable at higher AGI, dropping the refundable $360."
-us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Declared income too high for the ESCC and the NY child care credit. At $117,653 federal AGI, the $1,000 under-4 ESCC is reduced by only $693 to $307, and the NY CDCC is a refundable $360 (60% of $600) with no income cutoff."
-us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Applied the 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also counted 43 phase-out steps instead of 42 whole $1,000 steps over $75,000, giving an ESCC of $290.50 instead of $307."
-us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used the superseded ESCC formula, the greater of $100 or 33% of a $2,000 federal CTC ($660), and asserted there is no phase-out. The enhanced credit is $1,000 reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360), and it understated AGI by deducting the full $33,000 capital loss instead of the $3,000 cap."
-us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Assumed the ESCC is fully phased out at this income, when the $1,000 under-4 credit is only reduced to $307. It also omitted the refundable NY child and dependent care credit of $360 on the $9,600 of childcare expenses."
-us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Assumed the ESCC is phased out when $307 remains ($1,000 less 42 × $16.50). It also used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360)."
-us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Checked only the NY earned income credit and ignored the refundable Empire State Child Credit ($307 for the child under 4) and the refundable NY child and dependent care credit ($360).
-us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also omitted the $307 Empire State Child Credit for the child under 4.
-us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"Applied the old rule that excludes children under 4 from the ESCC, but the 2026 enhanced credit pays $1,000 for a child under 4, which phases down to $307 here. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Took the full $1,000 under-4 ESCC and never applied the $16.50-per-$1,000 reduction for federal AGI above $75,000 (42 steps, $693), which leaves $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)."
-us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount (33% of a $1,000 CTC) instead of the $1,000 under-4 credit phased down to $307. It estimated the NY CDCC at $300 instead of $360 and added a $63 NYC school tax credit that is not a refundable state credit for this household."
-us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the enhanced $1,000 under-4 credit phased down to $307. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied the old $100 ESCC minimum instead of the $1,000 under-4 credit reduced by $693 to $307. It also used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360)."
-us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied the 110% NY CDCC percentage that applies only at the lowest incomes, giving $660, instead of the 60% percentage at this income, which gives $360. It also used the superseded $330 ESCC amount instead of the phased-down $307."
-us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported zero without reasoning. The correct derivation adds the $307 Empire State Child Credit ($1,000 under-4 amount less the $693 phase-out) and the $360 refundable NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the $1,000 under-4 credit phased down to $307. It also left out the refundable $360 NY child and dependent care credit on the $9,600 of childcare costs."
-us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the enhanced $1,000 under-4 credit phased down to $307. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also assumed the ESCC is phased out when $307 remains after the $693 reduction.
-us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_082,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Set the ESCC equal to the full $2,200 federal CTC instead of the $1,000 under-4 amount reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable NY credits. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit generated by the $9,600 of childcare expenses."
-us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,Looked only for NY EITC eligibility and returned zero. It missed the $307 Empire State Child Credit and the $360 refundable NY child and dependent care credit.
-us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Applied a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also wrongly concluded that no child credit remains, when the $1,000 under-4 ESCC is only reduced to $307."
-us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Took the unreduced $1,000 under-4 ESCC and skipped the $693 phase-out for federal AGI above $75,000, which leaves $307. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also counted 43 phase-out steps instead of 42 whole $1,000 steps, giving an ESCC of $290.50 instead of $307."
-us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"Applied the old ESCC age floor that excludes children under 4, but the 2026 enhanced credit pays $1,000 for a child under 4, which phases down to $307 here. It also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also over-reduced the ESCC to $290.50 by counting 43 steps instead of 42 whole $1,000 steps over $75,000, which gives $307."
-us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also treated the ESCC as fully phased out when $307 of the $1,000 under-4 credit remains."
-us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Wrongly treated the NY child and dependent care credit as nonrefundable; it is refundable at $360 (60% of $600). It also assumed there is no ESCC at this income, when $307 remains after the $693 phase-out."
-us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). This $240 gap accounts for almost all of the miss; its $312 ESCC is close to the correct $307 ($1,000 less 42 × $16.50)."
-us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,Found no qualifying refundable credits. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit.
-us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Tied the ESCC to a federal CTC it wrongly called fully phased out; the federal phase-out starts at $200,000. The enhanced ESCC is $1,000 for a child under 4 less $16.50 per $1,000 over $75,000, which gives $307. It also ignored the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Applied the old $100 ESCC minimum on the false premise that the federal CTC is phased to zero, instead of the $1,000 under-4 credit reduced to $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)."
-us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Computed the ESCC as 33% of a $2,000 federal CTC ($660) under the superseded formula with no phase-out, instead of the $1,000 under-4 amount less $693, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)."
-us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Checked only the NY EITC and ignored the refundable Empire State Child Credit ($307) and the refundable NY child and dependent care credit ($360).
-us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as nonrefundable and absorbed by tax liability, but it is refundable in full at $360 (60% of $600). It also omitted the $307 Empire State Child Credit for the child under 4."
-us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Computed the ESCC from a recomputed 2017-law federal CTC and fell back to the $100 minimum. The 2026 credit is a flat $1,000 for a child under 4, reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)."
-us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Asserted that income exceeds the ESCC range. The $1,000 under-4 credit is only reduced by $693 to $307 at $117,653 federal AGI, and the model also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Correctly started from the $1,000 under-4 ESCC and the $75,000 head-of-household threshold. It then used a 5% phase-out rate instead of $16.50 per whole $1,000 (1.65%), which wiped out the credit instead of leaving $307, and it also omitted the refundable $360 NY child and dependent care credit."
-us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Checked only the NY earned income credit. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit.
-us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Its $313 covers only an ESCC-sized amount, close to the correct $307. It omitted the refundable NY child and dependent care credit of $360 (60% of the $600 federal CDCC) that brings the total to $667."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It computed about $5,891 of pre-credit tax on $105,454 using rates higher than the 2026 HOH schedule, which gives $5,442.63 of main tax plus $156.51 of supplemental tax. It then subtracted about $396 for New York's child and dependent care credit. That credit is refundable in New York, so it doesn't reduce tax before refundable credits. The model also kept the $68 SALT refund in NY AGI instead of subtracting it."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used the correct 2026 HOH main schedule, but it computed the supplemental recapture by flattening taxable income at 5.4%, which gave only $49.75 instead of $156.51. Above $107,650 of NY AGI, the recapture uses the higher rate of the bracket that starts at $107,650. It also kept the $68 SALT refund in NY AGI and deducted the $8 IRA contribution, which is nondeductible with a workplace plan at this income."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It set NY taxable income equal to a mis-summed federal taxable income of $62,149 after the $14,600 federal standard deduction. The correct path starts from NY AGI of $117,593.62 and subtracts the $11,200 NY HOH standard deduction and the $1,000 dependent exemption, giving $105,393.62. Its $5,649 doesn't follow from any NY schedule applied to its own $62,149 base."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a rounded $5,200 with no derivation. The 2026 HOH schedule on $105,393.62 yields $5,442.63, and NY AGI above $107,650 adds $156.51 of supplemental tax. Its figure leaves out the supplemental tax and understates the bracket tax as well."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed the head as single with the $8,000 standard deduction and no $1,000 dependent exemption, instead of head of household with the $11,200 deduction. It also added the $2,289 short-term gain on top of the $3,000 capped loss, even though that gain is already netted into the capital loss. After applying the pre-2026 single schedule, it took about $1,145 off its $6,221 figure without any credit that applies at this income."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used single status with an $8,000 standard deduction and no dependent exemption, instead of the HOH $11,200 deduction plus the $1,000 exemption. It also skipped the $181 traditional 401(k) exclusion. It then subtracted an unexplained ~$1,050 for household and child-care credits, although the household credit is phased out at this AGI and NY's child-care credit is refundable."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used an $8,000 standard deduction for head of household, but New York's HOH amount is $11,200, and it left out the $1,000 dependent exemption. Together these pushed taxable income up to about $109,654. It then applied a rough 5.5–6% marginal estimate instead of the 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) plus the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,386 was essentially right, but its main tax came to $5,808.15 instead of $5,442.63. It cut only the lower brackets and didn't apply the 2026 rate of 5.4% across the $20,900–$107,650 HOH bracket. Its $81.40 supplemental recapture also falls short of the correct $156.51."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated the head as a single filer and itemized $9,917. Head-of-household status gives an $11,200 standard deduction, which exceeds the $9,328 of NY itemized deductions, plus a $1,000 dependent exemption. It also used the outdated 5.85% single schedule and subtracted $120 for New York's child care credit, which is refundable and doesn't reduce tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used single status and itemized $9,917 instead of taking the HOH $11,200 standard deduction plus the $1,000 dependent exemption. It then applied a 6.25% single bracket. Finally, it cut about $1,393 for a household credit that is fully phased out when NY AGI is about $117,600."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it estimated the tax at about $5,400 using a loose 4–5.5% range. The 2026 HOH schedule gives exactly $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its $86,949 federal AGI deducts essentially the entire $30,711 net capital loss, but the annual deduction is capped at $3,000. It then used a ~$10,000 standard deduction and $2,000 of exemptions, although New York allows only the $1,000 dependent exemption. It also subtracted the Empire State child credit, which is refundable."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of NY itemized deductions, far more than the actual $9,328. Because $9,328 is less than the $11,200 HOH standard deduction, the standard deduction applies, so taxable income is $105,393.62 rather than its $95,971. It also used bracket widths that don't match 2026, applied 5.5% as the top rate, and left out the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It took the $11,200 standard deduction but dropped the $1,000 dependent exemption. Its pre-credit figure of $5,731.77 doesn't match the 2026 schedule, which gives $5,442.63 of main tax plus $156.51 of supplemental tax. It then subtracted $120 for NY's child and dependent care credit as if it were nonrefundable, though New York's version is refundable."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $2,000 of exemptions, but New York has no personal exemption for the head, only $1,000 per dependent, so it understated taxable income by $1,000. It also left out the $156.51 supplemental recapture required when NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 HOH amount and left out the $1,000 dependent exemption. It then subtracted a $30 household credit, which isn't available when NY AGI is about $117,600. It also left out the $156.51 supplemental recapture."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It taxed only wage income after the standard deduction. That leaves out $20,480 of dividends and $100 of interest, which net of the $3,000 capital-loss deduction lift NY AGI to $117,593.62. It also missed the $156.51 supplemental tax that applies once NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of $84,178 is about $21,216 below the correct $105,393.62, which matches subtracting the $21,208 of employer-sponsored insurance premiums plus the $8 IRA from income. Neither reduces the listed gross wages in this computation. Taking the premiums out also dropped NY AGI below $107,650 and removed the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 is about $21,140 below the correct $117,593.62, which matches subtracting the $21,208 of ESI premiums as a pre-tax wage exclusion that doesn't apply here. That pushed NY AGI below the $107,650 supplemental-tax threshold and dropped the $156.51 recapture. Its taxable income of $79,254 also doesn't match the stated $11,200 deduction and $1,000 exemption."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $6,040.50 is $441 above the correct figure: the 2026 HOH schedule on $105,393.62 gives $5,442.63, plus $156.51 of supplemental tax. The overstatement matches taxing NY AGI as a single filer with the $8,000 standard deduction and no dependent exemption, instead of HOH with the $11,200 deduction and the $1,000 exemption."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,220 is about $1,379 below the correct $5,599.14, which matches taking roughly $21,000 out of taxable income, the size of the $21,208 ESI premiums. Those premiums don't reduce income here. Removing them also dropped NY AGI below $107,650 and eliminated the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,591 is $2,008 below the correct $5,599.14, which means about $37,000 of taxable income is missing from the correct $105,393.62. That gap matches treating items such as the $21,208 ESI premiums or the uncapped $30,711 capital loss as deductions, when only a $3,000 capital loss is allowed."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct $11,200 HOH deduction and $1,000 exemption. However, its $5,533 reflects pre-2026 rates: 5.5% on the $20,900–$107,650 bracket instead of 2026's 5.4%, with matching higher rates on the lower brackets. It also left out the $156.51 supplemental tax triggered by NY AGI above $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $5,599.14 reference."
+us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It built $27,966 of NY itemized deductions from two errors. It included about $4,700 of NY income tax, which New York requires adding back when itemizing. It also claimed $13,349 of medical expenses, treating the ESI premiums as deductible, when only $1,965 of out-of-pocket medical exists and it falls below the 7.5% floor. Because itemized deductions are only $9,328, the $11,200 HOH standard deduction applies; the model also skipped the $1,000 dependent exemption and subtracted NY's refundable child-care credit."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave a rough estimate of $5,150 without applying the 2026 HOH schedule. That schedule yields $5,442.63 on $105,393.62 of taxable income, and NY AGI above $107,650 adds $156.51 of supplemental tax. Its figure leaves out the supplemental tax and understates the bracket tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no derivation, and its $7,777 is more than New York would charge even with zero deductions. The 2026 HOH schedule on all of the $117,593.62 NY AGI plus the recapture comes to well under $7,000. The correct path is the $11,200 standard deduction and $1,000 exemption, then the 2026 HOH schedule, giving $5,442.63 plus $156.51 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it applied the pre-2026 HOH rates of 4%, 4.5%, 5.25% and 5.5%, which give $5,548. The 2026 rates of 3.9%, 4.4%, 5.15% and 5.4% give $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,954.80 corresponds to roughly $55,000 of taxable income, about half the correct $105,393.62. That is the scale of deducting both the uncapped $30,711 capital loss and the $21,208 ESI premiums, though only a $3,000 loss is allowed and the premiums don't reduce income. It also dropped the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of roughly $106,394 leaves out the $1,000 dependent exemption, which brings the correct figure to $105,393.62. Its $5,815 also exceeds the 2026 HOH result, $5,442.63 of main tax plus $156.51 of supplemental tax, because it applied rates higher than 2026's 5.4% in the $20,900–$107,650 bracket."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It took the $11,200 standard deduction but left out the $1,000 dependent exemption. Its $5,977.08 also overstates the 2026 HOH tax. Applying 3.9%, 4.4%, 5.15% and 5.4% to $105,393.62 gives $5,442.63, and the supplemental recapture adds only $156.51."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its taxable income of $92,961.65 implies about $23,632 of NY itemized deductions, which is consistent with treating the $21,208 ESI premiums as deductible medical expenses. Actual NY itemized deductions are $9,328.22, so the $11,200 HOH standard deduction applies, giving $105,393.62 of taxable income."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,458 is essentially the 2026 HOH main-schedule tax alone ($5,442.63 on $105,393.62). It left out the $156.51 supplemental recapture that applies because NY AGI of $117,593.62 exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted $120 for the NY child and dependent care credit as if it were nonrefundable, but New York's credit is refundable and doesn't reduce tax before refundable credits. Its pre-credit figure of $5,665.24 also exceeds the correct $5,599.14 ($5,442.63 under the 2026 HOH schedule plus $156.51 of supplemental tax)."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $92,961.65 implies about $23,632 of NY itemized deductions, which is consistent with treating the $21,208 ESI premiums as deductible medical expenses. Actual NY itemized deductions are $9,328.22, so the $11,200 HOH standard deduction applies and taxable income is $105,393.62."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave a rounded $5,500 without applying the actual schedule. The 2026 HOH rates (3.9%, 4.4%, 5.15%, 5.4%) on $105,393.62 give $5,442.63, and NY AGI above $107,650 adds $156.51 of supplemental tax, which its estimate leaves out."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 subtracts the $21,208 ESI premiums, which don't reduce income here, so the correct NY AGI is $117,593.62. It used a $12,100 HOH standard deduction instead of $11,200 and subtracted NY's refundable child-care credit as if it were nonrefundable. With AGI understated, it also missed the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it applied the pre-2026 formula of $901 plus 5.5% over $20,900. The 2026 rates of 3.9%, 4.4%, 5.15% and 5.4% give $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It itemized deductions even though the $11,200 HOH standard deduction exceeds the $9,328 of NY itemized deductions. It also taxed the result on the single schedule ($4,271 plus 6% over $80,650) instead of the 2026 HOH schedule. It left out the $156.51 supplemental tax as well."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It left out the $1,000 dependent exemption, giving $106,386 of taxable income instead of $105,393.62. It then used pre-2026 rates with the wrong HOH bracket widths (5,400 and 3,000 instead of 4,850 and 3,250). It also skipped the $156.51 supplemental tax. Its $5,598 comes from offsetting errors rather than from the $5,442.63 main tax plus $156.51 supplemental."
+us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted $2,000 of exemptions, but New York has no personal exemption for the head, only $1,000 per dependent. It also didn't subtract the $68 SALT refund from NY AGI. It then took off $120 for NY's child care credit, which is refundable and doesn't reduce tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $5,599.14 reference."
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it used a rate of about 5.7% for the bracket containing that income. Under the 2026 HOH schedule, the $20,900–$107,650 bracket is taxed at 5.4%, which gives $5,442.63. It also didn't compute the separate $156.51 supplemental recapture."
+us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the $33,000 long-term capital loss as wiping out most of the income. The net capital loss is limited to a $3,000 deduction, so NY AGI is still $117,593.62. It also used a single $8,000 deduction and pointed to a CTC, which doesn't reduce NY tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used $8,000 as the HOH standard deduction, but New York's HOH amount is $11,200. It also applied pre-2026 rates, including 6.25% above $107,650. Finally, it subtracted $120 for NY's child-care credit, which is refundable and doesn't reduce tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $30,711 net capital loss, giving federal AGI of $89,951, when the annual deduction is capped at $3,000 and federal AGI is $117,661.12. It used single status with the $8,000 deduction instead of HOH with $11,200. Its final $4,755.44 is higher than its own $4,292.89 pre-credit figure, so the closing step is arithmetically inconsistent."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $139,586 ignores the $3,000 capital-loss deduction. It then subtracted an invented 'federal deduction adjustment' and used single status with the $8,000 deduction instead of HOH with $11,200. It also subtracted the refundable Empire State Child Credit, a household credit that is phased out at this income, and a NY CTC that doesn't exist."
+us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the pre-2025 ESCC formula (33% of a $1,000 pre-TCJA base = $330) instead of the enhanced $1,000 under-4 credit, which a $16.50-per-$1,000 phase-out above $75,000 reduces to $307. It also guessed about $150 for the care credit instead of applying the 2026 §606(c-2) percentage of about 29.35% to $3,000 of expenses, which gives $880.55."
+us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120). From 2026, §606(c-2) instead applies an AGI-based percentage of about 29.35% directly to the $3,000 expense cap, giving $880.55. It also applied the ESCC phase-out as a continuous 1.65% of the excess ($703.79) rather than $16.50 per full $1,000 (42 × $16.50 = $693), so it got $296.21 instead of $307."
+us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said NY has no refundable child credit. The refundable Empire State Child Credit is $307 after phase-out, and the refundable 2026 §606(c-2) child care credit is $880.55, so refundable credits total $1,187.55, not $0."
+us,scenario_082,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Said no refundable NY credits apply at this income, with no derivation. The $1,000 under-4 ESCC is only partly phased out ($307 remains), and the 2026 §606(c-2) child care credit is refundable at $880.55."
+us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Made up a $110 under-4 ESCC amount instead of the enhanced $1,000 credit less the $16.50-per-$1,000 phase-out above $75,000 ($307). It also wrongly treated the NY child care credit as nonrefundable above about $50K, dropping the $880.55 refundable §606(c-2) credit."
+us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Assumed the ESCC and care credit fully phase out or become nonrefundable at this income. In fact the $1,000 under-4 ESCC keeps $307 after a $693 reduction, and the 2026 §606(c-2) care credit refunds $880.55 (about 29.35% of $3,000)."
+us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Concluded income was too high for any NY refundable credit. At federal AGI of about $117.7K the under-4 ESCC is still $307, and the 2026 §606(c-2) child care credit is still $880.55."
+us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used the pre-2026 care credit of 20% of the federal $600 ($120). §606(c-2) instead applies about 29.35% to $3,000 of expenses, giving $880.55. It also rounded the ESCC phase-out up to 43 increments ($709.50) instead of counting 42 full $1,000 increments ($693), so it got $290.50 instead of $307."
+us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Computed the ESCC as 33% of a $2,000 federal CTC ($660) with no phase-out. The 2026 under-4 credit is a flat $1,000 cut by $16.50 per $1,000 of AGI over $75,000, giving $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) percentage on $3,000 ($880.55), and understated AGI by subtracting the full $33,000 long-term loss rather than the $3,000 cap."
+us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Assumed the ESCC is fully phased out and left out the care credit entirely. The $1,000 under-4 ESCC keeps $307, and the 2026 §606(c-2) refundable child care credit adds $880.55."
+us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed the ESCC is fully phased out, but the $1,000 under-4 credit keeps $307 after 42 × $16.50 = $693 of reduction. It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Only checked the NY EITC and missed the two refundable credits that apply: the Empire State Child Credit ($1,000 less a $693 phase-out = $307) and the 2026 §606(c-2) child care credit ($880.55)."
+us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Left out the $307 Empire State Child Credit for the under-4 child. It also computed the care credit as 20% of the federal $600 ($120) instead of applying the 2026 §606(c-2) percentage of about 29.35% to $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied the old rule that excludes children under 4 from the ESCC. Since 2025 a child under 4 gets a $1,000 credit, which phases down to $307 here. It also left out the refundable 2026 §606(c-2) child care credit of $880.55."
+us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Gave the full $1,000 ESCC without the $16.50-per-$1,000 phase-out above $75,000 of federal AGI, which removes $693 and leaves $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) percentage on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the old $330 ESCC instead of the phased $1,000 under-4 credit ($307) and guessed $300 for the care credit instead of the §606(c-2) amount of $880.55. It also added an NYC School Tax Credit, which does not apply because the household is not stated to be in NYC."
+us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Counted only a $330 ESCC from the pre-2025 33% formula. It missed the enhanced $1,000 under-4 credit phased to $307, and left out the $880.55 refundable 2026 child care credit."
+us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Used the $100 ESCC minimum instead of the enhanced $1,000 under-4 credit less a $693 phase-out ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Applied the 110% low-income multiplier from the old federal-linked care credit table ($660), which does not fit this AGI or the 2026 §606(c-2) structure (about 29.35% of $3,000 = $880.55). It also used the old $330 ESCC instead of the phased $1,000 under-4 credit ($307)."
+us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported zero refundable credits with no derivation, missing the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 §606(c-2) child care credit."
+us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Reported only the old $330 ESCC rather than the $1,000 under-4 credit reduced by $693 to $307, and left out the refundable 2026 child care credit of $880.55."
+us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Counted only a $330 ESCC from the pre-2025 formula, instead of the $307 left from the $1,000 under-4 credit after phase-out. It also left out the $880.55 §606(c-2) child care credit."
+us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Used the old 20%-of-federal care credit ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also assumed the ESCC phases out completely, though $307 of the $1,000 under-4 credit remains."
+us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no substantive answer to score against the $1,187.55 reference."
+us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Treated the enhanced ESCC as 100% of the federal CTC ($2,200). The under-4 credit is a flat $1,000 cut by $16.50 per $1,000 of AGI over $75,000, giving $307. It also left out the $880.55 refundable §606(c-2) child care credit."
+us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable NY credits, missing that the under-4 child gets a $307 ESCC after phase-out and that the $9,600 of child care expenses produce a refundable 2026 §606(c-2) credit of $880.55."
+us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Looked only at the NY EITC and assumed nothing else applied. The ESCC ($307 after phase-out) and the refundable 2026 child care credit ($880.55 on $3,000 of expenses) both apply from the listed child and childcare facts."
+us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120) instead of the §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also wrongly said no ESCC remains, though $307 of the $1,000 under-4 credit survives."
+us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Gave the full $1,000 under-4 ESCC without the $16.50-per-$1,000 phase-out above $75,000, which reduces it to $307. It also left out the $880.55 refundable 2026 child care credit."
+us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Used the pre-2026 care credit of 20% of the federal $600 ($120) instead of the §606(c-2) amount of $880.55. It also rounded the ESCC phase-out up to 43 increments ($290.50) instead of 42 full $1,000 increments ($307)."
+us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the old rule that excludes children under 4 from the ESCC. The enhanced credit gives a 1-year-old $1,000, phased to $307 here. It also left out the refundable 2026 §606(c-2) child care credit of $880.55."
+us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also rounded the ESCC phase-out to 43 increments ($290.50) instead of 42 ($307)."
+us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Wrongly said the ESCC is fully phased out, though $307 of the $1,000 under-4 credit remains. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,Wrongly treated the NY dependent care credit as nonrefundable and the child credit as unavailable. The 2026 §606(c-2) care credit refunds $880.55 and the under-4 ESCC refunds $307.
+us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Got close on the ESCC ($312 vs. $307) but used the old 20%-of-federal care credit ($120). §606(c-2) applies about 29.35% to the $3,000 expense cap, giving $880.55."
+us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Found no refundable NY credits, missing the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 child care credit."
+us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Wrongly treated the federal CTC as fully phased out at about $117.7K AGI (the head-of-household phase-out starts at $200,000). It then dropped the ESCC, which is $1,000 less $693 = $307, and left out the $880.55 refundable §606(c-2) care credit."
+us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used the $100 ESCC minimum on the false premise that the federal CTC is phased out. The enhanced under-4 ESCC is $1,000 less $693 = $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) amount of $880.55."
+us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $660 ESCC (33% of a $2,000 CTC) instead of the $1,000 under-4 credit cut by $16.50 per $1,000 above $75,000 ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) amount of $880.55."
+us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Only checked the EITC and missed both refundable NY credits that apply: the $307 under-4 ESCC and the $880.55 2026 child care credit.
+us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as fully absorbed by tax liability, but the 2026 §606(c-2) credit is refundable at $880.55. It also left out the refundable $307 ESCC for the under-4 child."
+us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no substantive answer to score against the $1,187.55 reference."
+us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Used the old 2017-law-CTC ESCC path with its $100 minimum, missing the enhanced flat $1,000 under-4 credit, which phases down to $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Assumed income exceeds the ESCC range, but the $1,000 under-4 credit keeps $307 at about $117.7K AGI. It also left out the $880.55 refundable 2026 child care credit."
+us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Applied a 5% ESCC phase-out rate. The actual reduction is $16.50 per full $1,000 above $75,000 (42 × $16.50 = $693), leaving $307. It also left out the $880.55 refundable §606(c-2) child care credit."
+us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Looked only at the NY EITC and missed the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 child care credit.
+us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Got close on the ESCC ($313 vs. $307) but left out the refundable 2026 §606(c-2) child care credit. That credit applies about 29.35% to $3,000 of expenses and adds $880.55."
us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero."
us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output."
us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation."
@@ -6905,11 +7084,12 @@ us,scenario_083,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_el
us,scenario_083,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a Texas Medicaid pathway for low-income young adults. Age 20 and MAGI below the income limit do not establish eligibility without a covered category, and this person qualifies through none."
us,scenario_083,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The answer treated age, low income, and Texas residence as sufficient without applying Texas's categorical eligibility rules. The head is a nondependent adult with no qualifying pathway, so the engine assigns `medicaid_category` NONE."
us,scenario_083,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated low income and age 20 as sufficient for WIC without applying WIC's categorical restriction. A nonpregnant, nonpostpartum, nonbreastfeeding 20-year-old adult is outside every qualifying WIC category, so passing the income test does not confer eligibility."
-us,scenario_083,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model made inconsistent arithmetic, first understating Social Security tax as $436.10 and then deriving a $538.33 subtotal before submitting the unrelated value $540.33. Applying 6.2% and 1.45% to $7,050 yields $437.10 and $102.22, totaling $539.33."
-us,scenario_083,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model named the correct wage base and payroll-tax rates but did not perform the stated multiplication correctly. A combined 7.65% tax on $7,050 produces $539.325, which rounds to $539.33, not $540.90."
-us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's submitted $498.95 contradicts its own component calculation and its statement that both Social Security and Medicare apply. It also misstated $7,050 × 1.45% as $102.175; the correct components are $437.10 and $102.22, totaling $539.33."
-us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,The model correctly derived $539.33 and then improperly changed it to $538 under the label of rounding. Ordinary currency rounding preserves $539.33; it does not reduce the result by $1.33.
-us,scenario_083,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $649.62 is inconsistent with applying employee Social Security and Medicare taxes to the stated $7,050 wages. The correct 7.65% combined tax on that wage base is $539.33, so the answer used an inflated base or rate unsupported by the household facts."
+us,scenario_083,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The base and rates were right ($7,050 at 6.2% and 1.45%), but 6.2% of $7,050 was miscalculated as $436.10 instead of $437.10, giving a subtotal of $538.33. The model then added an arbitrary $2 as 'standard practice' rounding and submitted $540.33 instead of $539.33."
+us,scenario_083,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model added the $542 FLSA overtime premium to the $7,050 of wages and taxed $7,592. The prompt says gross wages already include overtime, so the premium was counted twice. It then submitted $575, which does not even match its own $581 result; the correct figure is 7.65% of $7,050, or $539.33."
+us,scenario_083,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly limited the base to $7,050 of wages at 6.2% plus 1.45% with no Additional Medicare or Texas payroll tax. It then summed wrong: $437.10 + $102.225 is $539.33, not the $540.90 it submitted."
+us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model miscalculated Medicare as $102.175 instead of $102.225, getting $539.275. It then cut that to an unexplained $498.95 by citing a non-existent 'netting' of the only two components. No deduction applies, and the answer is the full $437.10 + $102.225 = $539.33."
+us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,"The model computed Social Security of $437.10 and Medicare of $102.225, reaching the correct $539.33. It then 'rounded' that down to $538, which is not rounding at all, and submitted the wrong figure."
+us,scenario_083,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model said it was taxing the head's wages but gave no figures. Its $649.62 equals 7.65% of about $8,492, a wage base well above the $7,050 of gross wages. Taxing the correct base gives $437.10 Social Security plus $102.22 Medicare, or $539.33."
us,scenario_083,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $3,960 of rental income as self-employment income and applied the 92.35% adjustment and 15.3% tax rate to it. Ordinary rental income is excluded from net earnings from self-employment, leaving no self-employment earnings subject to the $400 threshold and yielding $0 of self-employment tax."
us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"It checked only the gross and net income tests and never tested the $6,620 bank balance against Texas's $5,000 SNAP resource limit, which makes the household ineligible. Its own work gave about $122 a month, or $1,469 a year, but it submitted $2,718, which none of its arithmetic supports."
us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"It noticed the $6,620 exceeds the standard resource limit but wrongly said Texas BBCE waives the asset test. Texas BBCE keeps a $5,000 asset limit, so the household is ineligible. It then abandoned its own $1,430 calculation for an unexplained $244.50 a month."
@@ -6935,75 +7115,77 @@ us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli
us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and blindness as sufficient for Medicaid while leaving the required income, resource, and coverage-group tests unapplied. PolicyEngine assigns no Medicaid category because those statuses alone do not establish eligibility."
us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded SSDI from Medicaid MAGI merely because it considered the benefit nontaxable. Medicaid MAGI includes Social Security benefits through its program-specific adjustment, producing 1.56 times FPL rather than approximately $8,405 and placing the head above the expansion limit."
us,scenario_084,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model calculated expansion MAGI as approximately $8,405 by omitting the head's Social Security disability income. The Medicaid MAGI computation yields 1.56 times FPL, so the head exceeds North Carolina's expansion threshold and has no other qualifying category."
-us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model treated the instruction to hold disability constant through the year as meeting the 24-month SSDI waiting period. That instruction covers only 12 months of the tax year. The number of prior SSDI months is not listed, so it is 0, and the 24-month disability pathway fails for a 62-year-old."
-us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as giving Medicare eligibility by itself and skipped the rule requiring 24 months of disability-benefit entitlement. The Head's months of SSDI receipt are not listed, so they are 0, which leaves a 62-year-old ineligible."
-us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI requirement, then dropped it and concluded that being disabled and receiving SSDI is enough. No fact establishes 24 months of receipt. That count is not listed and is therefore 0, so the Head, at 62, is not Medicare eligible."
-us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI waiting period but treated disability held constant for the year as satisfying it. A full year of receipt is only 12 months, and prior SSDI months are not listed, so they are 0. The disability pathway fails."
-us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model said outright that it was 'assuming established disability benefit receipt'. That invents SSDI history the prompt tells it not to infer. With months of SSDI receipt at 0, the 24-month requirement is not met, and at age 62 the Head is not Medicare eligible."
-us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated the SSDI disability pathway as automatic once disability benefits are paid. It left out the rule requiring 24 months of benefit entitlement. That count is not listed, so it is 0, and the Head is not eligible."
-us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model wrote that the Head 'has been receiving' SSDI, which invents a history of receipt long enough to meet the 24-month waiting period. The prompt gives no months of SSDI receipt, so that count is 0. A 62-year-old without 24 months of entitlement does not qualify."
-us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model mentioned the SSDI waiting period but never checked whether it had passed. The months of SSDI receipt are not listed, so they are 0, well short of the 24 months required. The Head, at 62, is therefore not eligible."
-us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model inferred 24 months of SSDI receipt from the Head's age of 62. Age says nothing about how long benefits have been paid. The months of receipt are not listed, so they are 0, and the 24-month disability pathway is not met."
-us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability provision without its main condition: 24 months of entitlement to SSDI. With months of SSDI receipt at 0 (not listed), a 62-year-old does not qualify under that provision."
-us,scenario_084,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine makes every disabled SSDI recipient eligible for Medicare. PolicyEngine actually requires age 65, ESRD, or at least 24 months of SSDI receipt. The months count is not listed and is therefore 0, so none of these apply."
-us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model concluded that disabled status plus SSDI income makes the Head Medicare eligible. It never applied the 24-month SSDI entitlement requirement, which fails because months of receipt are not listed and default to 0."
-us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model treated SSDI income by itself as qualifying for Medicare and ignored the 24-month entitlement waiting period. Months of SSDI receipt are 0 because they are not listed, so the Head, at 62, is not eligible."
-us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model equated receiving SSDI with Medicare eligibility and skipped the rule requiring 24 months of disability-benefit entitlement. The months count is not listed, so it is 0, and the Head is not eligible."
-us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model concluded that disabled status plus SSDI income gives Medicare eligibility without applying the 24-month SSDI waiting period. Prior months of receipt are 0 because they are not listed, so a 62-year-old does not qualify."
-us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt and disability as enough for Medicare and left out the 24-month entitlement requirement. With months of SSDI receipt at 0 (not listed), the disability pathway fails."
-us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's rules cover anyone under 65 who is disabled and receiving SSDI. PolicyEngine's disability pathway actually requires at least 24 months of SSDI receipt. That count is not listed and is therefore 0, so the Head is not eligible."
-us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model cited disabled and surviving-spouse status as grounds for Medicare. Neither qualifies anyone without 24 months of entitlement to disability benefits, and that also applies to disabled widow(er) benefits. The months of receipt are not listed and are therefore 0, so the Head, at 62, is not eligible."
-us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly said someone aged 62 needs disability-based entitlement, then treated having SSDI income as proof of that entitlement. Medicare eligibility requires 24 months of SSDI entitlement. That count is not listed and is therefore 0, so the Head is not eligible."
-us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated disability status plus SSDI income as meeting PolicyEngine's disability rule. That rule requires at least 24 months of SSDI receipt, and the months count is 0 because it is not listed. The Head, at 62, is therefore not eligible."
-us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility to someone under 65 based only on receiving SSDI. It never applied the 24-month entitlement requirement, which fails because months of SSDI receipt are not listed and default to 0."
-us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model used the disability pathway without checking its requirement of 24 months of SSDI entitlement. The months of SSDI receipt are 0 because they are not listed, so the Head is not eligible at 62."
-us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model assumed PolicyEngine's disability-based Medicare pathway applies to any disabled SSDI recipient. That pathway requires at least 24 months of SSDI receipt. The months count is not listed and is therefore 0, so the pathway does not apply."
-us,scenario_084,head_medicare_eligible,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model counted disabled status plus SSDI income as meeting PolicyEngine's disability-based Medicare rule. That rule requires 24 months of SSDI receipt, and the months count is 0 because it is not listed. The Head is therefore not eligible."
-us,scenario_084,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model gave disability-based Medicare to a disabled SSDI recipient without applying the rule requiring 24 months of benefit entitlement. Months of receipt are not listed, so they are 0, which leaves a 62-year-old ineligible."
-us,scenario_084,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model treated reported disability status and SSDI income as meeting the disability-based Medicare rule. That rule turns on 24 months of SSDI receipt, and the prompt gives none, so the count is 0. The Head is not eligible."
-us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model asserted the Head had been on SSDI 'for sufficient duration', but no listed fact gives any duration of receipt. Months of SSDI receipt are 0 because they are not listed, so the 24-month requirement fails."
-us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model declared the 24-month SSDI waiting period satisfied without any fact to support it. Months of SSDI receipt are not listed, so they are 0, and a 62-year-old without 24 months of entitlement is not Medicare eligible."
-us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model mentioned the SSDI waiting period but assumed it had passed. Months of SSDI receipt are not listed, so they are 0, short of the 24 months required. The Head is therefore not eligible."
-us,scenario_084,head_medicare_eligible,grok-4.7,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's Medicare rule as 'age 65 or older or if disabled'. Disability counts only after at least 24 months of SSDI receipt. That count is not listed and is therefore 0, so a 62-year-old does not qualify."
-us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt by someone under 65 as immediately qualifying for Medicare and left out the 24-month entitlement requirement. Months of receipt are 0 because they are not listed, so the Head is not eligible."
-us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model inferred that the 24-month waiting period was met from the sizes of the Head's disability and SSDI payments. Dollar amounts say nothing about how long benefits have been paid. The months of SSDI receipt are not listed, so they are 0, and the Head is not eligible."
-us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model treated the instruction that disability and SSDI status are constant through the year as establishing 24 months of SSDI receipt. That instruction covers only the 12-month tax year. Prior months of receipt are not listed, so they are 0, and the 24-month requirement fails."
+us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model named the 24-month SSDI waiting period but treated it as met because disability is constant for the year. The months of SSDI receipt are unlisted, so they are 0. Even a full year of receipt is only 12 months, short of the 24 the disability pathway requires, and at 62 the head fails the age-65 test."
+us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as enough for Medicare before age 65. It skipped the requirement of 24 months of SSDI entitlement, and the unlisted months of receipt are 0. At 62, the head also fails the age-65 test."
+us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI rule but then concluded that disability plus SSDI receipt alone confers eligibility. The months of SSDI receipt are unlisted, so they are 0 and the waiting period is not met. The head, at 62, is also under the age-65 threshold."
+us,scenario_084,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare entitlement and ignored the 24-month waiting period. The unlisted months of SSDI receipt are 0, so the disability pathway fails, and age 62 fails the age-65 test."
+us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model acknowledged the 24-month SSDI waiting period but treated constant disability status as proof it was met. The unlisted months of receipt are 0, and one year of receipt gives at most 12 months, so the head is not eligible at 62."
+us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model explicitly assumed established SSDI receipt to clear the 24-month waiting period. The prompt forbids inferring unlisted facts, so the months of receipt are 0. The disability pathway fails, and age 62 is below 65."
+us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI receipt as satisfying the Medicare disability pathway and never applied the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, so the 62-year-old head is not eligible."
+us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model asserted that the head has been receiving SSDI past the 24-month waiting period, but no duration is given. Unlisted inputs are 0, so the waiting period is unmet, and the 62-year-old head fails the age-65 test."
+us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model assumed the SSDI waiting period had already passed. The months of SSDI receipt are unlisted and therefore 0, which fails the 24-month requirement, and the head is 62, under 65."
+us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt at age 62 as implying at least 24 months of receipt, which is an inference the prompt prohibits. The unlisted months of SSDI receipt are 0, so the 24-month waiting period is unmet and the head is not Medicare eligible."
+us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability provision on disability status and SSDI income alone. It omitted the 24-month SSDI entitlement requirement, which fails because the unlisted months of receipt are 0, and age 62 fails the age-65 test."
+us,scenario_084,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model claimed that under PolicyEngine rules, being disabled with SSDI income confers Medicare eligibility. In fact, PolicyEngine requires 24 months of SSDI receipt for under-65s, and that count is 0 here because it is unlisted."
+us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI income as enough for Medicare and ignored the 24-month waiting period. The months of SSDI receipt are unlisted, so they are 0, and the 62-year-old head is not eligible."
+us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model concluded that receiving SSDI income alone qualifies the head for Medicare. Under-65 eligibility requires 24 months of SSDI receipt, and the unlisted months of receipt are 0."
+us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as qualifying for Medicare and skipped the 24-month entitlement requirement. Months of receipt default to 0 because they are unlisted, so the 62-year-old head fails both the disability and age-65 pathways."
+us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated disability and SSDI income as automatic Medicare eligibility. It ignored the 24-month SSDI waiting period, which is unmet because the unlisted months of receipt are 0, and age 62 is under 65."
+us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as qualifying for Medicare without applying the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, so the 62-year-old head is not eligible."
+us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's rule as disabled plus SSDI income. The actual under-65 test requires at least 24 months of SSDI receipt, and that count is 0 here because it is unlisted."
+us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model counted disabled and surviving-spouse status as Medicare eligibility at 62. Surviving-spouse status creates no Medicare pathway. The disability pathway requires 24 months of SSDI receipt, and the unlisted months of receipt are 0."
+us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly noted that age 62 does not qualify, then treated disabled status plus SSDI income as SSDI-based Medicare entitlement. That entitlement requires 24 months of SSDI receipt, and the unlisted months of receipt are 0."
+us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated disability and SSDI income as enough under PolicyEngine's disability rules. PolicyEngine requires 24 months of SSDI receipt for under-65 eligibility, and the months of receipt are 0 because they are unlisted."
+us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model granted disability-based Medicare from SSDI income alone and never applied the 24-month waiting period. That period fails because the unlisted months of SSDI receipt are 0, and the head is 62."
+us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied the under-65 disability pathway without checking its 24-month SSDI entitlement requirement. The unlisted months of SSDI receipt are 0, so the head is not Medicare eligible."
+us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model treated being a disabled SSDI beneficiary as meeting the modeled disability pathway. That pathway requires at least 24 months of SSDI receipt, and the unlisted months of receipt are 0, so the 62-year-old head fails it."
+us,scenario_084,head_medicare_eligible,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI income as PolicyEngine's disability-based Medicare classification. That classification requires 24 months of SSDI receipt, which is 0 here because it is unlisted."
+us,scenario_084,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model granted disability-based Medicare from current SSDI receipt and ignored the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, and age 62 fails the age-65 test."
+us,scenario_084,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied the disability-based rule using only disability status and reported SSDI income. The rule requires at least 24 months of SSDI receipt, and the unlisted months of receipt are 0."
+us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model asserted that the head had received SSDI for a sufficient duration, but no duration appears in the facts. Unlisted numeric inputs are 0, so the 24-month waiting period fails and the 62-year-old head is not eligible."
+us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model claimed the 24-month SSDI waiting period was satisfied without any stated duration of receipt. The unlisted months of receipt are 0, so the disability pathway fails, and age 62 is below 65."
+us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model assumed the head had served the SSDI waiting period. The months of SSDI receipt are unlisted and therefore 0, short of 24, so the 62-year-old head is not Medicare eligible."
+us,scenario_084,head_medicare_eligible,grok-4.7,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's rule as age 65 or older, or disabled. Under 65, PolicyEngine requires at least 24 months of SSDI receipt, which is 0 here because it is unlisted, so disability status alone does not confer eligibility."
+us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,The model treated disability plus SSDI income as qualifying for Medicare under 65 and omitted the 24-month SSDI entitlement requirement. That requirement fails because the unlisted months of receipt are 0.
+us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model assumed the 24-month waiting period from the size of the disability and SSDI income amounts. Benefit amounts say nothing about duration, and the unlisted months of SSDI receipt are 0, so the 62-year-old head fails the disability pathway."
+us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read 'constant throughout the year' as satisfying the 24-month SSDI requirement. Constant status over one year gives at most 12 months, and the unlisted months of receipt are 0, so the head is not Medicare eligible at 62."
us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Sonnet derived monthly net income of $1,737.59 but never compared it to the SNAP net income limit of 100% FPL (about $1,340/month for one person), which that figure fails by roughly 30%, and it ignored both NC's 200%-FPL BBCE gross cap (about $2,680/month against its own $2,769) and the $56,500 in bank assets against the elderly/disabled resource limit of roughly $4,650. Its own benefit formula returned $0 because 30% of net income ($521.28) exceeds the one-person maximum allotment ($298), after which it discarded that result and substituted an unsupported $183/month figure attributed to a conditional minimum benefit, a rule that provides $23-$24, not $183. It further inflated the excess medical deduction by double-counting the same $3,600 premium (listed once as health insurance premiums excluding Medicare Part B and again as other health insurance premiums) and by including $1,054 of non-prescription over-the-counter costs, which are not deductible medical expenses."
us,scenario_084,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"DeepSeek asserted the household passes the net income test after the standard and medical deductions, but net income of about $1,738/month exceeds the one-person net income limit near $1,340/month, gross income of $2,769/month exceeds North Carolina's 200%-FPL BBCE gross cap of about $2,680/month, and $56,500 in countable resources exceeds the elderly/disabled asset limit of roughly $4,650. Having wrongly cleared eligibility, it applied the $23/month minimum allotment for one- and two-person households, a floor that attaches only after a household qualifies; an income-ineligible household receives $0, not $276."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference exactly: taxable SS $14,051, AGI $45,574, deductions of $18,150 plus the $6,000 senior deduction, taxable income $21,424, and tax of $2,323. It then submitted 940, a number its own explanation never derives."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the full $27,426 of Social Security into AGI instead of the $14,050.60 taxable portion, and it used a made-up $27,750 deduction. It then zeroed the tax with nonrefundable credits the filer does not get. The Credit for the Elderly is reduced to zero because the $13,375 of nontaxable Social Security alone exceeds the $5,000 initial amount. AGI of $45,574 is above the Saver's Credit income limit."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its AGI of $45,574 is correct. However, it used about $17,000 for the aged standard deduction instead of $16,100 + $2,050 = $18,150, so its total deduction was $23,000 rather than $24,150. It also ended the 10% bracket at the 2025 figure of $11,925 instead of the 2026 figure of $12,400. That produced $2,470 instead of $2,322.76."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at about $20,705 without applying the formula, which caps it at 0.85 × ($45,236 − $34,000) + $4,500 = $14,051. That inflated AGI to $52,228. It also used $15,750 + $2,000 instead of the 2026 amounts of $16,100 + $2,050. Together these pushed taxable income to $28,478 and the tax to $3,169."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning matches the reference: AGI $45,574, deductions of $24,150 including the $6,000 senior deduction, taxable income $21,424, and tax of about $2,323. It then submitted 1,170, which contradicts its own calculation."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the OBBBA $6,000 senior deduction entirely. It also used 2025-level deductions ($15,000 + $2,000 = $17,000) instead of $16,100 + $2,050 = $18,150, and 2025 brackets. It subtracted the $8 traditional 401(k) contribution even though there are no wages for it to reduce. The result was taxable income of $28,566 instead of $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its derivation used a $17,000 aged standard deduction with no $6,000 senior deduction, giving $3,189. It then submitted 626, backed only by a vague reference to 'additional itemized deduction consideration' that it never computed. The correct $24,150 of deductions gives $2,322.76."
-us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It took only a $17,460 standard deduction ($15,400 + $2,060) and left out the $6,000 senior deduction. It also understated the 2026 base deduction, which is $16,100. That overstated taxable income at $28,114 instead of $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law: a $9,800 aged standard deduction and a 15% second bracket. OBBBA made the TCJA brackets and larger standard deduction permanent, so those do not apply in 2026. It also overstated taxable Social Security at $19,668.60 instead of $14,050.60."
-us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $45,574 is correct. It subtracted only a $17,050 aged standard deduction, however, leaving out the $6,000 senior deduction and understating the base-plus-aged amount of $18,150. Taxable income came out at $28,524 instead of $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, $18,150 standard deduction, and 2026 brackets are all correct, but it left out the $6,000 OBBBA senior deduction for filers 65 and older. That added $6,000 × 12% = $720, turning $2,322.76 into $3,042.83."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying an $8,100 standard deduction, a $5,050 personal exemption, and a 15% bracket. For 2026, OBBBA's permanent rules apply instead: a $16,100 + $2,050 standard deduction, the $6,000 senior deduction, and 10%/12% brackets."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It started from $58,949 of gross taxable income, which counts the full $27,426 of Social Security as taxable instead of the $14,050.60 that the provisional-income formula allows. It also never applied the full $24,150 of deductions, which includes the $6,000 senior deduction."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its AGI of $45,573.60 is correct, but it applied pre-TCJA baseline rules with a personal exemption and the older, smaller standard deduction. For 2026, OBBBA's $16,100 + $2,050 standard deduction, $6,000 senior deduction, and 10%/12% brackets apply."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied rules that assume the TCJA had expired, with a personal exemption and a 15% bracket, instead of OBBBA's permanent 10%/12% brackets and the $24,150 of deductions that includes the senior deduction. It also subtracted the $8 traditional 401(k) contribution even though there are no wages for it to reduce."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no working. The correct calculation leaves $21,423 of taxable income after $24,150 of deductions and a tax of $2,322.76. A $0 answer implies the model treated the $31,523 of pension, IRA, 401(k), and interest income, plus the taxable Social Security, as fully offset, which the deductions do not do."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $45,573.60 is correct, but it subtracted only a $17,450 aged standard deduction and left out the $6,000 senior deduction. It also used the 2025 top of the 10% bracket ($11,925) instead of $12,400."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, with a standard deduction, aged addition, and personal exemption totaling $15,650 and higher post-TCJA rates. For 2026, OBBBA's $24,150 of deductions and 10%/12% brackets apply."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 deduction amounts ($15,000 + $2,000 = $17,000) and left out the $6,000 senior deduction. The correct 2026 total is $16,100 + $2,050 + $6,000 = $24,150. It also used the 2025 10% bracket limit of $11,925."
-us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $17,000 aged standard deduction and 2025 brackets. It left out the $6,000 OBBBA senior deduction and understated the 2026 standard deduction of $16,100 + $2,050, giving taxable income of $28,573.60 instead of $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted only $17,100 ($15,450 + $1,650) and left out the $6,000 senior deduction. It also understated the 2026 base-plus-aged standard deduction, which is $18,150. Taxable income came out at $28,474 instead of $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It concluded there was no liability because the household has no wages and no dependents. It ignored that the $20,720 pension, $7,568 IRA distribution, $1,560 401(k) distribution, $1,675 of interest, and $14,050.60 of taxable Social Security are ordinary taxable income. After $24,150 of deductions, that leaves $21,423 of taxable income."
-us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the income was offset by the standard deduction and by retirement contributions or credits. The contributions total only $8 of traditional 401(k), with no wages for them to reduce, and no nonrefundable credits apply. The $24,150 of deductions still leaves $21,423 taxable, for $2,322.76 of tax."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its reference to 'deductions and exemptions' shows it applied rules that assume the TCJA had expired, with personal exemptions and higher rates. For 2026, OBBBA's $24,150 of deductions and 10%/12% brackets apply, giving $2,322.76, not about $4,200."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its AGI of $45,574 is correct, but it assumed the TCJA had expired, with an aged standard deduction plus personal exemption of $15,889 and a 15% bracket. For 2026, OBBBA's permanent rules apply: $16,100 + $2,050 + a $6,000 senior deduction, and 10%/12% brackets."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the TCJA sunset: an $8,550 basic deduction, a $2,100 aged amount, a $5,450 personal exemption, and a 15% bracket. OBBBA made the TCJA structure permanent and added the $6,000 senior deduction, so the correct deductions total $24,150 with 10%/12% brackets."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules: a standard deduction, aged addition, and personal exemption totaling $15,350, with higher brackets. For 2026, OBBBA's $24,150 of deductions, including the $6,000 senior deduction, and 10%/12% brackets apply."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the TCJA reversion, with a $10,631 aged standard deduction and a 15% bracket, instead of OBBBA's permanent 10%/12% brackets and $24,150 of deductions. It also taxed Social Security at about the 85% maximum ($23,264) instead of the $14,050.60 the provisional-income formula gives."
-us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI, $18,150 standard deduction, and 2026 brackets are correct, but it left out the $6,000 OBBBA senior deduction. That overstated taxable income by $6,000 and the tax by $720 ($3,043 instead of $2,322.76)."
-us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It taxed 85% of Social Security ($23,312) instead of applying the provisional-income formula, which gives $14,050.60. It also left out the $6,000 senior deduction. It then concluded the tax was near zero, even though its own figures leave more than $40,000 of taxable income after a $16,850 deduction."
-us,scenario_085,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as only 0.85 × ($45,236 − $34,000) = $9,550.60. It dropped the $4,500 base amount (the lesser of $4,500 or half of benefits) that the worksheet adds, so it understated AGI at $41,073.60 and its explanation reached $1,782.83. The value it submitted, $3,680, matches neither that explanation nor the correct $2,322.76."
-us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It taxed the full 85% maximum of Social Security ($23,312.10) instead of the $14,050.60 the formula gives. It subtracted only about $15,750 in deductions and left out the $6,000 senior deduction. It then submitted 8,587.50, which its own figure of about $4,500 does not support."
-us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It taxed 85% of Social Security instead of $14,050.60 and miscounted AGI as $68,548. It took only a $16,100 deduction, with no aged addition or senior deduction. It then invented a '$1,600 Senior Bonus credit' and a '$2,100 survivor credit', although the OBBBA senior provision is a $6,000 deduction and no survivor credit exists."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got every step right: taxable SS $14,051, AGI $45,574, deductions of $18,150 + $6,000, and tax of about $2,323. It then submitted $940, which contradicts its own calculation and has no basis in the reasoning."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It counted all $27,426 of Social Security in AGI instead of the $14,051 taxable portion, and it used a made-up $27,750 standard deduction. It then wiped out the tax with a Credit for the Elderly and a Saver's Credit, but neither applies. The elderly credit is zero because the nontaxable SS and an AGI above $17,500 eliminate it. The Saver's Credit is unavailable because there is no earned income and AGI is over the limit."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It called the survivor benefits 'largely nontaxable,' but provisional income of $45,236 is above $34,000, so about $14,051 is taxable and AGI is $45,574. Its $1,550 estimate is not tied to the correct taxable income of $21,423.74 after the $24,150 in deductions."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got AGI right but used a $17,000 standard deduction instead of 2026's $18,150 ($16,100 + $2,050 aged). Its total deduction of $23,000 should have been $24,150. It also used the 2025 10% bracket top of $11,925 instead of $12,400, which produced $2,470 instead of $2,322.85."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at about $20,705 instead of applying the two-tier formula, which gives min(85% × 27,426, 0.85 × 11,236 + 4,500) = $14,051. That inflated AGI to $52,228, and it also used a $15,750 base standard deduction instead of $16,100."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning computed taxable income of about $21,424 and tax of about $2,320 from the correct $24,150 deduction. It then submitted $1,170, a value that contradicts its own derivation."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and used a $17,000 standard deduction instead of 2026's $18,150. It also subtracted an $8 traditional 401(k) contribution even though there were no wages to defer, and it used the 2025 $11,925 bracket. The result was taxable income of $28,566 instead of $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $17,000 standard deduction and never applied the $6,000 senior deduction, which produced $3,189. It then submitted $626 with no supporting calculation. The correct deductions are $18,150 + $6,000, giving taxable income of $21,423.74 and tax of $2,322.85."
+us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It got AGI right but deducted only $17,460 (a $15,400 base plus $2,060 aged) and left out the $6,000 OBBBA senior deduction. It also used a $12,050 10% bracket, which gave $3,133 instead of tax on $21,423.74 of taxable income."
+us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It overstated taxable Social Security at $19,668.60 instead of $14,051. It also used a pre-TCJA $9,800 standard deduction and a 15% bracket, but OBBBA made the TCJA $16,100 + $2,050 standard deduction and the 12% bracket permanent, and it added the $6,000 senior deduction, which the model also left out."
+us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It got AGI right but used a $17,050 standard deduction and left out the $6,000 senior deduction. That put taxable income at $28,524 instead of $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It got AGI ($45,573.60), the $18,150 standard deduction, and the brackets right, but it left out the $6,000 OBBBA senior deduction for filers 65 and older with MAGI under $75,000. Its taxable income of $27,423.60 should have been $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used an $8,100 standard deduction, a personal exemption, and a 15% bracket. OBBBA made the TCJA standard deduction ($18,150 with the aged addition) and the 12% bracket permanent and added the $6,000 senior deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $27,426 of survivor benefits in gross income ($58,949) instead of the $14,051 taxable portion from the two-tier provisional-income formula. It also left out the $6,000 senior deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got AGI right but applied a 'pre-TCJA baseline' with a personal exemption. Under OBBBA, 2026 keeps the TCJA standard deduction of $18,150 for a 65+ single filer, adds the $6,000 senior deduction, and uses the 10%/12% brackets, which gives $2,322.85."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed TCJA expired and used a personal exemption and the 15% bracket, and it subtracted an $8 401(k) contribution with no wages to defer. It never applied the permanent $18,150 standard deduction or the $6,000 OBBBA senior deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave $0 without any calculation. The pension, IRA, 401(k), interest, and $14,051 of taxable Social Security produce AGI of $45,574 and taxable income of $21,423.74 after $24,150 in deductions, and no nonrefundable credit offsets the $2,322.85 tax."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It got AGI right but deducted only $17,450, left out the $6,000 senior deduction, and used the 2025 $11,925 bracket top. Its taxable income of $28,123.60 should have been $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed TCJA expired and used a $15,650 package of standard deduction, aged addition, and personal exemption with pre-TCJA rates. Under OBBBA, the deduction is $18,150 plus the $6,000 senior deduction, taxed at 10%/12%."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 standard deduction of $17,000 and 2025 brackets, and it left out both the 2026 inflation increase to $18,150 and the $6,000 OBBBA senior deduction. Its taxable income was $28,573.60 instead of $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It got AGI right but deducted only about $17,000 and never applied the $6,000 senior deduction. It also used 2025 brackets (10% to $11,925) instead of 2026's $12,400."
+us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It got AGI right but used a $17,100 standard deduction instead of $18,150 and left out the $6,000 OBBBA senior deduction. Its taxable income of about $28,474 should have been $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It reasoned that having no wages and no dependents means no tax, but pension, IRA, 401(k), interest, and taxable Social Security are all ordinary taxable income. After $24,150 in deductions, they leave $21,423.74 of taxable income and $2,322.85 of tax."
+us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed retirement contributions and credits offset the income, but the $8 401(k) and $1 Roth amounts cannot be contributed without earned income, and no nonrefundable credit applies. Taxable income of $21,423.74 produces $2,322.85."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied 'deductions and exemptions,' which assumes TCJA expired and brings back the personal exemption and the 15% bracket. It never used the permanent $18,150 standard deduction plus the $6,000 senior deduction, and its $4,200 is far above the correct tax on $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got AGI right but assumed TCJA expired and used a $15,889 deduction-plus-exemption total with a 15% second bracket. OBBBA keeps the $18,150 standard deduction, adds the $6,000 senior deduction, and keeps the 12% bracket."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired and used an $8,550 base deduction, a $5,450 personal exemption, and a 15% rate. Under OBBBA, 2026 uses an $18,150 standard deduction plus the $6,000 senior deduction and the 12% bracket."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration brackets and a $15,350 deduction-plus-exemption total. OBBBA made the $18,150 aged standard deduction and the 10%/12% brackets permanent and added the $6,000 senior deduction, which gives taxable income of $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted the 85% maximum ($23,264) as taxable Social Security when the two-tier formula yields $14,051, and it subtracted an $8 401(k) contribution with no wages. It also assumed TCJA expired, using a $10,631 standard deduction and a 15% bracket, and it left out the $6,000 senior deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It got AGI, the $18,150 standard deduction, and the brackets right, but it left out the $6,000 OBBBA senior deduction. Its taxable income was $27,424 instead of $21,423.74."
+us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It overstated taxable Social Security as the full 85% ($23,312) instead of $14,051. It then claimed taxable income fell mostly within the 10% bracket and submitted $0, even though its own figures put taxable income around $44,600. The correct taxable income of $21,423.74 produces $2,322.85."
+us,scenario_085,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its taxable Social Security calculation left out the $4,500 first-tier amount (the lesser of $4,500 or 50% of benefits), giving $9,550.60 instead of $14,051 and understating AGI by $4,500. Its submitted $3,680 also contradicts its own computed $1,782.83."
+us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the 85% cap ($23,312) as taxable Social Security instead of the formula result of $14,051, and it left out the $6,000 senior deduction. It then submitted $8,587.50, which doesn't match its own roughly $4,500 calculation."
+us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It counted the 85% maximum of Social Security and inflated AGI to $68,548, when the correct figure is $45,574. It left out the aged standard deduction addition and the $6,000 senior deduction, and it made up a $1,600 'Senior Bonus credit' and a $2,100 survivor credit, neither of which exists."
us,scenario_085,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_085,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no wages and computed zero employee Social Security, Medicare, and state payroll taxes, but then submitted 1459 instead of carrying that zero into the value field. The submitted number is unrelated to its stated derivation."
@@ -7050,48 +7232,49 @@ us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_inc
us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category."
us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE."
us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security dependent benefits as the child's income, so the $10,900.40 taxable portion (85% cap, provisional income about $68,074) never reached the Head's AGI. It also capped educator expense at $300 and deducted $275 of auto loan interest. Its submitted $3,305 does not even match its own computed $1,925."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It deducted $275 of auto loan interest, but the facts establish no qualifying new vehicle, so no OBBBA vehicle-loan-interest deduction applies; that lowered taxable income to $48,137.40 instead of $48,412.90. It also topped the 2026 HoH 10% bracket at $17,000 instead of $17,700, which adds $14 of tax. Net result: $3,236.49 instead of $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included 100% of the $12,824 in SS benefits instead of the $10,900.40 taxable portion. It also subtracted $19,648 of employer-sponsored premiums and $3,120 of health premiums as above-the-line deductions, which are not allowed. It used single status with a $14,600 standard deduction instead of the $24,150 HoH amount. It then submitted $3,779, its tax before credits, without subtracting any CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It excluded the SS dependent benefits as the child's income, omitting $10,900.40 of taxable SS from the Head's AGI. It also used stale parameters: a $23,000 HoH standard deduction instead of $24,150, a $17,000 bracket top instead of $17,700, and a $2,000 CTC instead of $2,200. With the $300 educator cap, this gave $2,304."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It excluded the $10,900.40 of taxable SS benefits as the child's income and used a $23,000 HoH standard deduction and a $2,000 CTC. It then submitted $4,673, an inflated tax figure before credits, instead of its own after-CTC amount of $2,673."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It declared the SS benefits non-taxable 'at this income level' even though provisional income of $68,074 exceeds the $34,000 base, so $10,900.40 is taxable. It used a $24,500 standard deduction and a $2,000 CTC, then inflated its own $4,119 tax figure to $4,519 before subtracting the credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its computation is right except for the educator expense: it capped it at $300, but PolicyEngine's 2026 limit allows the full $337.50. That overstates taxable income by $37.50 ($48,450.40 vs $48,412.90) and tax by $4.50, giving $3,260.05."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA sunset for 2026, but OBBBA made the TCJA structure permanent. So it used pre-TCJA rules that no longer apply: an $8,300 single standard deduction, $10,600 of personal exemptions, a 15% bracket, and a $1,000 CTC. It also dropped the $10,900.40 of taxable SS as the child's income and filed as single instead of HoH."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the SS benefits as non-taxable even though provisional income exceeds $34,000, so the 85% cap makes $10,900.40 taxable. It used the stale $23,350 HoH deduction, $16,550 bracket and $2,000 CTC. It then replaced its own $2,271 result with an unexplained $3,892."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest, but no qualifying new vehicle is established, so the OBBBA vehicle-loan-interest deduction does not apply. That cut taxable income to about $48,137 instead of $48,412.90 and understated tax by about $33, giving $3,222."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It correctly included $10,900 of taxable SS but used a $23,100 HoH standard deduction instead of the 2026 $24,150. It subtracted a $2,000 CTC instead of $2,200, capped educator expense at $300, and rounded tax to $5,600, which gave $3,600."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (an $11,000 standard deduction, $9,600 of personal exemptions, a 15% bracket and a $1,000 CTC), but OBBBA made the TCJA structure permanent. It also subtracted the $1,560 health premium from wages and omitted the $10,900.40 of taxable SS."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,600.40 is right apart from the $300 educator cap. But it used a stale $23,100 HoH standard deduction instead of $24,150, which overstates taxable income by about $1,088. It also subtracted a $2,000 CTC instead of the 2026 $2,200, giving $3,591.05."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (a $12,200 standard deduction, $10,600 of personal exemptions, a 15% bracket and a $1,000 CTC), but OBBBA permanently kept the $24,150 HoH deduction, zero exemptions, the 12% bracket and the $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $16,500 standard deduction, $10,100 of personal exemptions, a 15% bracket and a $1,000 CTC, which OBBBA superseded. It also wrongly subtracted $19,648 of ESI premiums from the $62,000 gross wages."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted 100% of the $12,824 SS benefits as income instead of the $10,900.40 taxable portion and dropped the $337.50 educator deduction. It used a $15,300 standard deduction instead of the $24,150 HoH amount and a $2,000 CTC instead of $2,200, arriving at $1,782."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,560 is essentially correct, but it assumed the TCJA sunset: it used a pre-TCJA HoH standard deduction, two personal exemptions, the 15% bracket and a $1,000 CTC. OBBBA permanently set the 2026 values at $24,150, zero exemptions, 10%/12% brackets and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules (a $15,000 standard deduction, $10,100 of exemptions and a $1,000 CTC) that OBBBA superseded. It also subtracted the $19,648 of ESI premiums from gross wages, although the $62,000 wage input is already the taxable wage base."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. The correct path is AGI of $72,562.90 (including $10,900.40 of taxable SS), less the $24,150 HoH deduction, for taxable income of $48,412.90, tax of $5,455.55, and $3,255.55 after the $2,200 CTC. Its $3,043 is $212.55 too low, equivalent to understating taxable income by about $1,770 at the 12% rate."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $2,000 CTC instead of the 2026 $2,200. Its implied tax before credits of $5,678 is about $222 above the correct $5,455.55. That matches the stale $22,500 HoH standard deduction plus the $17,000 bracket top, instead of $24,150 and $17,700."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted personal exemptions, which OBBBA permanently set to zero, on top of the standard deduction. This understated taxable income and gave $1,888.74 instead of $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $72,600.40 AGI is right apart from the $300 educator cap, but it used the stale $22,500 HoH standard deduction instead of $24,150 and the $17,000 bracket top instead of $17,700. It also subtracted a $2,000 CTC instead of $2,200, giving $3,672.05."
-us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It treated the SS dependent benefits as the child's non-taxable income, but they are the Head's, and 85% ($10,900.40) is taxable. It also used a $22,500 HoH deduction instead of $24,150 and a $2,000 CTC instead of $2,200, arriving at $2,360."
-us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 and taxable income of about $48,212 are close to correct, but it computed tax of $6,347 on that income. The 2026 HoH schedule gives $1,770 + 12% of the excess over $17,700, about $5,431, so its bracket arithmetic overstated tax by roughly $900."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed nonrefundable credits fully offset the tax. The only nonrefundable credit is the $2,200 CTC, and taxable income of $48,412.90 generates $5,455.55 of tax, leaving $3,255.55, not $0."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted itemized medical expenses on top of the standard deduction, although the $3,060 of medical costs falls below the 7.5%-of-AGI floor and cannot be combined with the standard deduction. It also omitted the $10,900.40 of taxable SS, arriving at $2,682."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $10,900.40 of taxable SS and used a stale $22,500 HoH deduction. It then subtracted only $384.40 of CTC by reserving a 'refundable portion' first, but the nonrefundable CTC applies in full up to liability, so the full $2,200 is used."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included only $6,412 (50%) of the SS benefits as taxable. Provisional income of about $68,074 exceeds the $34,000 second base, so the 85% cap applies and $10,900.40 is taxable. This understated AGI and tax."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its components (taxable SS, $338 educator deduction, $24,150 HoH deduction, $2,200 CTC) match the correct path. Its $3,269 is $13.45 high, which matches taxing taxable income of about $48,413 with a $17,000 bracket top instead of the 2026 HoH $17,700 threshold (+$14)."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its brackets, $24,150 deduction and $2,200 CTC are right, but taxable income of $37,550 omits the $10,900.40 of taxable SS dependent benefits (85% cap at provisional income of about $68,074). It also capped educator expense at $300, understating tax by about $1,304."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It built taxable income from wages only, leaving out the $10,900.40 of taxable SS. It deducted 'limited mortgage interest' although only a mortgage balance is given, so mortgage interest is $0 and the $24,150 standard deduction applies alone. Its $4,523 does not follow from that base."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 is right, but it applied TCJA-expiration rules (a $12,389 deduction, $10,733 of personal exemptions, a 15% bracket and a $1,000 CTC). OBBBA made permanent the $24,150 HoH deduction, zero exemptions, the 12% bracket and the $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It correctly included $10,900 of taxable SS, but it used 'sunset' rules (a $12,400 HoH deduction, two $5,350 exemptions, a 15% bracket and a $1,000 CTC). OBBBA replaced these with the $24,150 deduction, zero exemptions, 10%/12% brackets and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It capped educator expense at $300 instead of deducting the full $337.50 and understated 85% of $12,824 as $10,890 instead of $10,900.40. It also computed $5,464 of tax on $48,440, where the correct brackets give $5,458.80, which left it at $3,264."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $72,562 AGI is right, but it applied post-TCJA-sunset rules (a $12,538 deduction, two $5,431 exemptions, a 15% bracket and a $1,000 CTC) that OBBBA superseded. The 2026 law uses the $24,150 HoH deduction, no exemptions, a 12% bracket and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It capped educator expense at $300 instead of deducting the full $337.50 and understated taxable SS as $10,890 instead of $10,900.40. That produced taxable income of about $48,440 instead of $48,412.90 and a result about $4.45 high."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no computation to evaluate against the correct $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest from AGI, but no qualifying new vehicle is established, so the OBBBA vehicle-loan-interest deduction does not apply. That reduced taxable income to $48,137.40 instead of $48,412.90 and understated tax by $33, giving $3,222.49."
-us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It filed as single with a $15,000 deduction and 2025 single brackets, but an unmarried parent with a 12-year-old qualifying child files head of household ($24,150 deduction, $17,700 10% bracket). It also omitted the $10,900.40 of taxable SS and used a $2,000 CTC instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as single with a $16,100 deduction, which pushed income into the 22% bracket, instead of head of household with $24,150. It deducted $275 of auto loan interest without a qualifying vehicle. It also applied only $500 of the CTC as nonrefundable, although the full $2,200 is used against liability before any refundable portion."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed married filing jointly although the Head is an unmarried single parent who files head of household. So it used a $31,200 MFJ deduction and the MFJ $23,850 bracket instead of $24,150 and $17,700, and it subtracted a $2,000 CTC instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the $32,200 MFJ standard deduction to a single parent who files head of household ($24,150). It also invented an 'SS dependent benefits exclusion' that zeroed out the $10,900.40 of taxable SS, driving tax to $0 instead of $3,255.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's income, so it left the $10,900.40 taxable portion out of Head's AGI. It also deducted $275 of auto-loan interest even though the facts show no qualifying vehicle loan. It then submitted $3,305, which contradicts its own $1,925 result."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion, the $24,150 HoH deduction and the $2,200 CTC right. However, it subtracted $275 of auto-loan interest (worth $33 of tax) that the facts don't support. It also ended the 10% bracket at $17,000 instead of $17,700, adding $14 and leaving a net shortfall of $19.06."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Filed as single with a $14,600 deduction instead of HoH with $24,150. It included the full $12,824 of benefits rather than the 85% taxable share, and subtracted employer and other health premiums as above-the-line deductions. It then submitted $3,779 without subtracting any CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Used married-filing-jointly treatment for a single parent who should file as head of household. It also built taxable income from wages alone, leaving out the $10,900.40 of taxable Social Security."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's and left the $10,900.40 taxable portion out of AGI. It also used a $23,000 HoH deduction instead of $24,150, a 10% bracket ending at $17,000 instead of $17,700, and a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Left the $10,900.40 of taxable Social Security out of AGI by assigning the benefits to the child, and used a $23,000 HoH deduction instead of $24,150. It then submitted $4,673, an unexplained pre-credit figure, rather than its own $2,673 after the CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Said the dependent benefits weren't taxable at this income, but provisional income of about $68,075 is above the $34,000 base, so $10,900.40 is taxable. It also used a $24,500 deduction instead of $24,150 and a $2,000 CTC instead of $2,200, then inflated its own $4,119 tax figure to $4,519 with no explanation."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Capped the educator expense deduction at $300, while the reference allows $337.50 above the line. That $37.50 of extra AGI, taxed at 12%, is the whole $4.50 gap; its 85% Social Security inclusion, $24,150 HoH deduction, brackets and $2,200 CTC all match the reference."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA expired in 2026 and applied pre-TCJA rules: a single-filer deduction, personal exemptions, a 15% bracket and a $1,000 CTC. Under the OBBBA-extended rules, a HoH filer gets $24,150, no exemptions, a 10%/12% schedule and a $2,200 CTC. It also left the $10,900.40 of taxable Social Security out by treating the benefits as the child's."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Treated the Social Security benefits as non-taxable, leaving out the $10,900.40 that the 85% rule includes. It used outdated figures ($23,350 deduction, 10% bracket ending at $16,550, $2,000 CTC), then replaced its own $2,271 result with an unexplained $3,892."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Matched the reference except that it subtracted $275 of auto-loan interest. The facts don't establish a qualified new, U.S.-assembled vehicle loan, so that deduction doesn't apply, and it cut tax by $33."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Included the taxable Social Security correctly but used a $23,100 HoH deduction instead of the 2026 figure of $24,150. It also used a $2,000 CTC instead of $2,200 and rounded the bracket tax roughly."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA rules ($11,000 deduction, $9,600 of exemptions, 15% bracket, $1,000 CTC) instead of the 2026 OBBBA rules. It subtracted the $1,560 health premium from wages, and it never included the $10,900.40 of taxable Social Security."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but used a $23,100 HoH deduction instead of $24,150, an older 10% bracket limit instead of $17,700, and a $2,000 CTC instead of the 2026 amount of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a $12,200 deduction, $10,600 of personal exemptions, a 15% bracket and a $1,000 CTC. The OBBBA-extended law gives a $24,150 HoH deduction, no exemptions, a 10%/12% schedule and a $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored insurance premiums from the $62,000 of gross wages, although the full $62,000 counts as employment income. It also applied TCJA-sunset rules (personal exemptions, 15% bracket, $1,000 CTC) instead of the 2026 HoH $24,150 deduction and $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Put the full $12,824 of benefits into gross income instead of the 85% taxable share of $10,900.40, used a $15,300 deduction instead of the HoH $24,150, and used a $2,000 CTC. The $1,782 doesn't even follow from those inputs."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Got AGI right at about $72,560 but assumed the TCJA expired in 2026. It applied personal exemptions, a pre-TCJA HoH deduction, a 15% bracket and a $1,000 CTC, instead of the $24,150 deduction, 10%/12% schedule and $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored premiums from the gross wages, and applied TCJA-sunset rules ($15,000 deduction, $10,100 of exemptions, $1,000 CTC) instead of the 2026 HoH $24,150 deduction and $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation. The correct path ($10,900.40 taxable SS, $72,562.90 AGI, $24,150 HoH deduction, $5,455.55 tax less $2,200 CTC) gives $3,255.55. Its $3,043 is $212.55 lower, which matches understating taxable income by about $1,771 at the 12% rate."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Applied a $2,000 CTC instead of $2,200. Its $3,678 matches a pre-credit tax of about $5,678, which comes from using the old $22,500 HoH deduction and a $17,000 10% bracket instead of the 2026 figures of $24,150 and $17,700."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Subtracted personal exemptions, which are zero under the OBBBA-permanent TCJA rules, on top of the HoH standard deduction. That cut taxable income well below $48,412.90 and left it $1,366.81 short."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but used the 2025 pre-OBBBA $22,500 HoH deduction instead of $24,150, a 10% bracket ending at $17,000 instead of $17,700, and a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's and left the $10,900.40 taxable portion out of Head's AGI. It also used a $22,500 HoH deduction instead of $24,150 and a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Got AGI ($72,562) and the $2,200 CTC right, but the bracket math was wrong. It computed $6,347 of tax on about $48,212, when the 10%/12% HoH schedule gives about $5,431; it also used a $24,350 deduction instead of $24,150."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the CTC and other nonrefundable credits wipe out the liability. In fact the $5,455.55 of pre-credit tax is far more than the only nonrefundable credit, the $2,200 CTC, so $3,255.55 of tax remains."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Built taxable income from wages alone, leaving out the $10,900.40 of taxable Social Security. It also subtracted medical expenses on top of the standard deduction, although the $3,060 total is below the 7.5%-of-AGI floor and can't be combined with the standard deduction."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Left the $10,900.40 of taxable Social Security out and used a $22,500 deduction instead of $24,150. It then cut the nonrefundable CTC to about $384 based on a supposed refundable portion, but the $2,200 CTC applies in full as a nonrefundable credit because tax exceeds it."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Limited taxable Social Security to 50% ($6,412). Provisional income of about $68,075 exceeds the $34,000 adjusted base, so the 85% tier applies and $10,900.40 is taxable."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Got the income and credit right, but its $3,269 is $14 above the reference. That gap equals ending the 10% HoH bracket at the 2025 limit of $17,000 instead of the 2026 limit of $17,700."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct $24,150 HoH deduction, 2026 brackets and $2,200 CTC, but built taxable income from wages only. It left out the $10,900.40 of Social Security that the 85% rule makes taxable for Head."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Left Social Security out of income and subtracted mortgage interest that the facts don't list (only a loan balance is given). With wage-only income its own method gives about $1,950, so the $4,523 also fails its own arithmetic."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $72,562 but applied TCJA-sunset rules: a $12,389 deduction, $10,733 of exemptions, a 15% bracket and a $1,000 CTC. It should have used the 2026 HoH $24,150 deduction, the 10%/12% schedule and the $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but applied TCJA-sunset rules ($12,400 deduction, two $5,350 exemptions, 15% bracket, $1,000 CTC) instead of the 2026 OBBBA HoH $24,150 deduction and $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Computed $5,464 of tax on $48,440 of taxable income, when the 2026 HoH schedule gives $5,458.80. It also took 85% of the benefits as $10,890 instead of $10,900.40 and capped the educator deduction at $300 rather than the reference's $337.50."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Got AGI right but applied TCJA-sunset rules ($12,538 deduction, $10,862 of exemptions, 15% bracket, $1,000 CTC) instead of the 2026 HoH $24,150 deduction, 10%/12% schedule and $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Capped the educator expense deduction at $300, while the reference allows $337.50, which raises tax by $4.50. Taking 85% of the benefits as $10,890 instead of $10,900.40 was a small offsetting slip; its HoH deduction, brackets and $2,200 CTC match."
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Subtracted $275 of auto-loan interest (as an above-the-line item). The facts don't establish a qualified new, U.S.-assembled vehicle loan, so no such deduction applies, and the extra deduction cut tax by $33; everything else matches."
+us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Filed as single with a $15,000 deduction and single brackets instead of head of household with a qualifying child ($24,150 deduction, 10% bracket up to $17,700). It also left out the $10,900.40 of taxable Social Security and used a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Filed as single ($16,100 deduction, 22% bracket) instead of HoH and deducted $275 of auto-loan interest that doesn't apply. It also allowed only $500 of nonrefundable CTC, but the full $2,200 is used against tax first because liability exceeds it."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated a parent and child as married filing jointly and applied MFJ Social Security thresholds and a $31,200 deduction. The correct status is head of household with a $24,150 deduction; it also used a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Applied the $32,200 MFJ deduction to a single parent and claimed the Social Security benefits are excluded, when 85% ($10,900.40) is taxable. It then reported zero tax even though $74,824 minus $32,200 is still positive."
us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed pre-credit tax of about $4,300 against a $2,200 CTC and then contradicted its own arithmetic by awarding $1,700, writing that the nonrefundable portion ""absorbs tax"" while still paying out the refundable cap. It applied only two of the three ACTC limitations — the $1,700 per-child cap and the 15%-of-earnings-over-$2,500 phase-in — and omitted the controlling leg of IRC 24(d)(1)(B), the amount of CTC blocked by the tax-liability limitation, which is $0 when liability ($5,455.55 here) exceeds the entire $2,200 credit."
us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model stated that ""tax liability is largely offset"" by the CTC and then treated the $1,700 refundable ceiling as the amount payable, inverting Schedule 8812: line 16a (total CTC minus the nonrefundable amount actually used) is $0 here, and the $1,700 cap can only limit that residual, never create one. It also used a stale $2,000 CTC rather than the 2026 OBBBA amount of $2,200, but the credit is fully absorbed at either figure since pre-credit tax is $5,455.55."
us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model derived the correct answer explicitly — ""the full $2,000 CTC would be absorbed by tax liability, leaving no refundable Additional Child Tax Credit"" — then overrode it with $1,700 by invoking unnamed ""itemized/above-the-line deductions reducing taxable income further."" Reaching any ACTC requires taxable income below roughly $21,280 under the 10%/12% HoH schedule, i.e. about $51,300 of deductions against $72,562.90 of AGI; the actual itemizable items here (mortgage interest on a $55,000 balance, Georgia income tax, and medical costs entirely below the 7.5%-of-AGI floor) fall far short of even the $24,150 standard deduction."
@@ -7113,51 +7296,52 @@ us,scenario_086,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"T
us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component amounts—$3,844 of Social Security tax and $899 of Medicare tax—but failed to add them correctly. Those components total $4,743, not $3,979, and no Georgia-specific adjustment changes that sum."
us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct $3,844 Social Security tax and $899 Medicare tax but submitted $3,813 instead of adding them. The stated components total $4,743."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used a $24,000 head-of-household standard deduction (the joint-filer amount) instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $1,749. It then dropped that result and submitted $2,732 based on an unexplained 'single-filer' treatment that matches none of its own arithmetic."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the $10,900.40 of taxable Social Security, but then also took $275 of auto-loan interest out of Georgia AGI; that is a below-the-line federal deduction and never enters AGI. It also used a $24,000 standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%, so it taxed $33,387 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied Georgia's repealed six-bracket schedule (1%–6%) to the full $62,000 of wages with no standard deduction and no dependent exemption. Georgia has had a flat tax since 2024; for 2026 the correct base is $61,662.50 − $15,000 − $5,000 = $41,662.50, taxed at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an $18,500 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $2,032.51. It then submitted $2,191 without any derivation."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the head as a single filer with a $12,000 standard deduction instead of the $15,000 head-of-household amount, used a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of 4.99%, which gave $2,324. It then reported an unsupported $1,869."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"From about $61,700 of Georgia AGI it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000, then applied 5.09% instead of the 2026 rate of 4.99%. That taxed $45,700 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It subtracted the full $12,824 of Social Security from an AGI of $61,662 that already excluded those benefits, counting the exclusion twice. It then used the repealed pre-2024 structure ($5,400 standard deduction, $2,700 personal exemption, $3,000 dependent exemption) and a 5.39% rate instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and 5.19% instead of 4.99%, which gave $2,372. It then inflated that to $2,765 with an unexplained 'standard deduction phase' adjustment that Georgia law does not have."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.09%. That uses the $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and 5.09% instead of the 2026 rate of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $72,600 and never subtracted the $10,900.40 of taxable Social Security, which Georgia fully exempts. It also used a $9,000 standard deduction plus two repealed $2,700 personal exemptions instead of the $15,000 head-of-household deduction and $5,000 dependent exemption, and a 5.29% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the repealed pre-2024 structure (a $5,400 standard deduction plus $2,700 personal exemptions for both head and child) and a 5.29% rate. The 2026 law gives a $15,000 head-of-household deduction and a $5,000 dependent exemption at 4.99%. It also reduced AGI to $60,140 by subtracting $1,560 of health insurance premiums, which are not an above-the-line deduction for a wage earner."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the repealed pre-2024 structure: a $5,700 standard deduction, a $2,700 filer exemption, and a $3,000 child exemption, which left $50,300 taxable. The correct 2026 amounts are a $15,000 head-of-household deduction and a single $5,000 dependent exemption. It also used 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the correct 4.99% rate but started from $62,000 of wages without the educator-expense adjustment. It then took a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus $6,700 of exemptions that include a taxpayer exemption Georgia no longer allows, instead of the single $5,000 dependent exemption. The result was $43,300 taxable instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $52,952.40 AGI subtracts the $19,648 of employer-sponsored insurance premiums from wages, which are not excluded from the listed gross wages here. It also failed to subtract the $10,900.40 of taxable Social Security for Georgia. On top of that it used a $15,800 head-of-household exemption, a $3,000 dependent exemption, and the 5.39% rate instead of $15,000, $5,000, and 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $2,690 matches taxing about $50,000 (the $62,000 of wages less only a roughly $12,000 standard deduction) at an old rate of about 5.39%. It omitted the $5,000 dependent exemption and used neither the $15,000 head-of-household deduction nor the 2026 rate of 4.99%. The correct result is ($61,662.50 − $20,000) × 4.99% = $2,078.96."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly reached Georgia AGI of $61,662, but then subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It also applied the old 5.39% rate instead of 4.99%, taxing $46,662 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a $52,952 AGI that wrongly subtracts the $19,648 of employer-sponsored insurance premiums from wages and still includes the $10,900 of taxable Social Security that Georgia exempts. It then took only $15,000 of combined deductions and exemptions instead of $20,000, and applied 5.29% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 is about $1,133 too high. It matches taxing federal AGI of about $72,562 less only a $12,000 deduction at roughly 5.29%, so the model kept the $10,900.40 of taxable Social Security that Georgia exempts. It also did not apply the $15,000 head-of-household deduction, the $5,000 dependent exemption, or the 4.99% rate."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $3,047 matches taxing about $56,500 at roughly 5.39%, which is federal AGI less about $16,000 of deductions. That means it left the $10,900.40 of taxable Social Security in Georgia income instead of subtracting it. The correct base is $41,662.50, taxed at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,432.32 equals tax on about $28,700 at 4.99%, roughly $13,000 less than the correct $41,662.50. So it subtracted about $33,000 of deductions and exemptions (the size of a $24,000 joint-level standard deduction plus exemptions) instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"From Georgia AGI of about $61,700 it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It then applied the old 5.39% rate instead of 4.99%, taxing $46,700 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly reached Georgia AGI of $61,662, but then used a $10,600 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely. It also applied 5.39% instead of 4.99%, taxing $51,062."
-us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"From the correct Georgia AGI of $61,662 it subtracted a $24,000 standard deduction (the joint-filer amount) instead of $15,000, plus about $8,400 of personal exemptions instead of the single $5,000 dependent exemption. That cut taxable income to $29,262, and it applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $1,639 equals tax on only about $32,800 at 4.99%, so it subtracted roughly $29,000 of deductions and personal exemptions from Georgia AGI. Georgia allows only $20,000 here: the $15,000 head-of-household deduction and the $5,000 dependent exemption, which leaves $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It computed Georgia tax from federal-style taxable income with federal standard-deduction and medical adjustments, so its $498 implies only about $10,000 of taxable income. Georgia instead subtracts just the $15,000 head-of-household deduction and the $5,000 dependent exemption from $61,662.50 of Georgia AGI, and taxes the $41,662.50 at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction instead of the $15,000 head-of-household amount, $4,500 of personal and dependent exemptions instead of the $5,000 dependent exemption, and 5.19% instead of the 2026 rate of 4.99%. It taxed $45,162 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It correctly removed taxable Social Security, but its $1,357.80 equals tax on only about $27,200 at 4.99%. That means it subtracted roughly $34,500 of deductions and exemptions instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption, which leave $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a $12,000 deduction instead of the $15,000 head-of-household standard deduction, a $4,000 dependent exemption instead of $5,000, and 5.09% instead of 4.99%. It taxed $45,662.40 instead of $41,662.50, and it also misstated the product ($45,662.40 × 5.09% = $2,324.22, not $2,323.22)."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied 5.19% instead of the 2026 rate of 4.99%, and its $1,903 implies about $36,670 of taxable income. That means it subtracted about $25,000 of deductions and exemptions instead of $20,000 (the $15,000 head-of-household deduction plus the $5,000 dependent exemption)."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.09%. That uses a $12,000 standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It had the correct 4.99% rate but claimed a $4,000 taxpayer exemption that Georgia repealed. It also used a $12,000 standard deduction instead of the $15,000 head-of-household amount, a $3,000 dependent exemption instead of $5,000, and capped the educator expense at $300, so it taxed $42,700 instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.19%. That uses a $12,000 deduction instead of the $15,000 head-of-household standard deduction, a $4,000 dependent exemption instead of $5,000, and 5.19% instead of the 2026 rate of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"From the correct $61,662 of Georgia AGI it subtracted a $12,000 deduction instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000. It then applied the scheduled 5.09% rate instead of the 4.99% rate in effect for 2026."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The correct computation is ($61,662.50 − $15,000 − $5,000) × 4.99% = $2,078.96. The model's $2,184 overstates tax by $105, which matches subtracting only about $17,900 of deductions and exemptions or applying a rate above 4.99%. It did not use the $15,000 head-of-household deduction, the $5,000 dependent exemption, and the 4.99% rate together."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It subtracted an $18,000 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely. It then applied 5.29% instead of 4.99%, taxing $43,662."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus $5,700 of combined personal ($2,700) and dependent ($3,000) exemptions. Georgia allows only a $5,000 dependent exemption and no taxpayer exemption. It also applied 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"From Georgia AGI of $61,700 it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It then applied 5.19% instead of 4.99%, taxing $46,700 rather than $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted the educator expense a second time even though federal AGI already includes that adjustment. It then applied the repealed pre-2024 structure ($5,400 standard deduction plus two $2,700 personal exemptions) and the 2024 rate of 5.49%, instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its taxable base of about $40,700 is close to the correct $41,662.50, but it applied 5.29% instead of Georgia's 2026 flat rate of 4.99%. That rate error inflates the tax by more than $120."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It had the correct 4.99% rate and $61,662 of Georgia AGI, but subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount. It also claimed two $2,700 personal exemptions ($5,400) instead of the single $5,000 dependent exemption, since Georgia no longer allows a taxpayer exemption, so it taxed $44,262 instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reduced Georgia AGI by $275 of auto-loan interest, which is a below-the-line federal deduction and not part of AGI. It also used an $18,500 head-of-household standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and 5.19% instead of 4.99%, and those errors happened to offset to $2,070.14."
-us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It subtracted only a $5,000 standard deduction and a $2,700 personal exemption from wages, instead of the $15,000 head-of-household deduction and the $5,000 dependent exemption, which left $54,000 taxable instead of $41,662.50. It also applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It cut Georgia AGI to $61,387 by subtracting $275 of auto-loan interest, which is not an AGI adjustment. It then used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a repealed $3,700 personal exemption, and a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It included all $12,824 of Social Security benefits in Georgia income, even though Georgia fully exempts Social Security (the $10,900.40 taxable portion is subtracted from federal AGI). It then used a $12,000 standard deduction, a repealed $2,700 personal exemption, a $3,000 dependent exemption, and a 5.12% rate instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said Social Security was excluded but started from gross income of $74,824, which includes the full $12,824 of benefits. It then subtracted a $23,100 standard deduction instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption. Its implied rate of about 4.57% on $51,700 also does not match Georgia's 4.99% flat rate."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used a $24,000 head-of-household standard deduction (the correct figure is $15,000), a $4,000 dependent exemption (correct: $5,000), and a 5.19% rate (correct: 4.99%). That produced its own figure of $1,749. It then submitted $2,732 under a 'single-filer' label, which contradicts its own calculation and matches no Georgia computation."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Subtracted $275 of auto loan interest, which is not a Georgia subtraction, so its Georgia AGI was $61,387 instead of $61,662.50. It also used a $24,000 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%. The overstated deduction drove taxable income down to $33,387 against the correct $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied Georgia's repealed six-bracket graduated schedule (1%–6%) to $62,000 of gross wages. It took no standard deduction or dependent exemption. Georgia's 2026 tax is a flat 4.99% on $41,662.50, which is Georgia AGI of $61,662.50 minus the $15,000 head-of-household deduction and the $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Used a 5.19% rate instead of Georgia's 2026 rate of 4.99%. Its $2,900 answer implies about $55,900 of taxable income, so it understated the $15,000 head-of-household standard deduction and $5,000 dependent exemption that bring taxable income to $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Used an $18,500 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $2,032.51. It then submitted $2,191 with no supporting arithmetic."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%, which gave $2,324. It then submitted $1,869 with no supporting arithmetic."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Correctly excluded Social Security, but subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000. It then taxed $45,700 at 5.09% instead of $41,662.50 at Georgia's 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Started from a federal AGI that already left out Social Security ($61,662), then subtracted the full $12,824 of benefits again. Georgia subtracts only the $10,900.40 of taxable Social Security that is in federal AGI. It also applied pre-2024 law: a $5,400 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.39%, instead of $15,000 + $5,000 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave about $2,372. It then submitted an unexplained, inflated $2,765."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia income, but subtracted only a $12,000 standard deduction (correct: $15,000 for head of household) and a $4,000 dependent exemption (correct: $5,000). It then applied 5.09% instead of 4.99%, taxing $45,662 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Never subtracted the $10,900.40 of taxable Social Security, so it started from $72,600 instead of Georgia AGI of $61,662.50. It also used a $9,000 standard deduction and two $2,700 personal exemptions instead of $15,000 + $5,000, at 5.29% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied pre-2024 Georgia law: a $5,400 standard deduction and $2,700 exemptions for the head and the child. Current law allows a $15,000 head-of-household standard deduction and a $5,000 dependent exemption with no personal exemption. It also used 5.29% instead of 4.99%, giving taxable income of $49,340 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used old-law deductions totaling $11,400: a $5,700 standard deduction plus $2,700 and $3,000 exemptions. The 2026 amounts total $20,000: a $15,000 head-of-household standard deduction plus a $5,000 dependent exemption. As a result it taxed $50,300 instead of $41,662.50, and at 5.09% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Used the correct 4.99% rate but started from wages without the $338 educator expense adjustment. It subtracted a $12,000 standard deduction instead of $15,000, and $6,700 of exemptions (a nonexistent personal exemption plus a dependent amount) instead of the single $5,000 dependent exemption. Taxable income came out at $43,300 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored insurance premiums from the stated wages. It also never removed the $10,900.40 of taxable Social Security from Georgia income. It then used a $15,800 'personal exemption', a $3,000 dependent exemption, and a 5.39% rate, instead of the $15,000 standard deduction, $5,000 dependent exemption, and 4.99% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $2,690 answer implies roughly $52,000–$54,000 of Georgia taxable income, which is consistent with taxing the $62,000 of wages after only about $8,000–$10,000 of deductions. The correct deductions are $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption) against $61,662.50 of Georgia AGI, taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It then taxed $46,662 at 5.39% instead of $41,662.50 at Georgia's 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Started from a $52,952 'federal AGI' built by subtracting the $19,648 employer-sponsored premiums from wages, and did not take out the taxable Social Security. It then subtracted only $15,000 of total deductions and exemptions instead of $20,000, and applied 5.29% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $3,212 answer implies about $60,000–$64,000 of taxable income, which exceeds Georgia AGI after deductions. That is consistent with leaving the taxable Social Security in Georgia income and applying little or no deduction, instead of taxing $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $3,047 answer implies roughly $56,500–$61,000 of taxable income. That is consistent with skipping most of the $15,000 head-of-household standard deduction and $5,000 dependent exemption, or with leaving taxable Social Security in Georgia income, instead of taxing $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,432.32 answer implies only about $26,500–$28,700 of taxable income, roughly $13,000–$15,000 below the correct $41,662.50. That is consistent with also subtracting the $19,648 employer-sponsored premiums or otherwise overstating deductions beyond the $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It also took Georgia AGI as $61,700 by capping educator expenses at $300. It then taxed $46,700 at 5.39% instead of $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $10,600 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption for the 12-year-old. It then applied 5.39% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Used a $24,000 standard deduction (the married-joint level) instead of the $15,000 head-of-household amount. It also added about $8,400 of personal and dependent exemptions instead of the single $5,000 dependent exemption, deducting $32,400 instead of $20,000. That cut taxable income to $29,262, which it then taxed at 5.19% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Its $1,639 answer implies only about $31,600–$32,800 of taxable income. That means it deducted roughly $9,000 more than the $15,000 head-of-household standard deduction plus $5,000 dependent exemption, and so understated the correct $41,662.50 taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Its $498 answer implies only about $10,000 of Georgia taxable income, which reflects grossly overstated deductions or adjustments. The correct computation subtracts only $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption) from $61,662.50 of Georgia AGI, then taxes $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used a $12,000 standard deduction instead of the $15,000 head-of-household amount and $4,500 of exemptions instead of the $5,000 dependent exemption. It then applied 5.19% instead of 4.99%, taxing $45,162 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Correctly excluded taxable Social Security, but its $1,357.80 answer implies only about $25,000–$27,000 of taxable income. That means it deducted roughly $35,000 instead of the $20,000 total from the $15,000 head-of-household standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Correctly reached $61,662.40 of Georgia income, but subtracted a $12,000 exemption instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000. It then applied a 5.09% rate instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $1,903 at 5.19% implies about $36,700 of taxable income. That means it deducted about $25,000 from $61,662 of Georgia AGI instead of $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption), and it used 5.19% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000. It then taxed $45,662 at a 5.09% rate instead of $41,662.50 at Georgia's 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Applied the correct 4.99% rate, but used a $12,000 standard deduction plus a $4,000 'taxpayer exemption' that Georgia no longer allows, and a $3,000 dependent exemption. Current law gives a $15,000 head-of-household standard deduction plus a $5,000 dependent exemption. It also capped educator expenses at $300, leaving taxable income at $42,700 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Used a $12,000 head-of-household deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000. It then applied 5.19% instead of 4.99%, taxing $45,662 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $12,000 head-of-household deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000. It then applied 5.09% instead of Georgia's 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Its $2,184 answer implies about $40,500–$43,800 of taxable income at the 5.0–5.4% rates in circulation. At 4.99% it implies about $43,800, about $2,100 above the correct $41,662.50, which is consistent with understating the $15,000 head-of-household standard deduction and $5,000 dependent exemption or using an outdated higher rate."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Used an $18,000 head-of-household standard deduction instead of $15,000 and entirely omitted the $5,000 dependent exemption for the child. It then applied a 5.29% rate instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used a $12,000 head-of-household standard deduction instead of $15,000, plus $5,700 of personal ($2,700) and dependent ($3,000) exemptions instead of the single $5,000 dependent exemption. It then applied 5.39% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Correctly subtracted taxable Social Security, but used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It then taxed $46,700 at 5.19% instead of $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Subtracted the $338 educator expense a second time, even though federal AGI already reflects it. It also applied pre-2024 law: a $5,400 standard deduction and two $2,700 personal exemptions, instead of the $15,000 head-of-household deduction and $5,000 dependent exemption. Finally, it used the 2024 rate of 5.49% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Reached roughly the right taxable base (about $40,700 against $41,662.50) after excluding Social Security. However, it applied a 5.29% flat rate instead of Georgia's 2026 rate of 4.99%, which overstated tax by roughly $70."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Applied the correct 4.99% rate to the correct $61,662 of Georgia AGI. However, it used a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus two $2,700 personal exemptions instead of a single $5,000 dependent exemption. Taxable income came out at $44,262 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Removed an extra $275 of auto loan interest, which put Georgia AGI at $61,387 instead of $61,662.50. It also used an $18,500 head-of-household standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Applied pre-2024 law: a $5,000 standard deduction and a $2,700 personal exemption, with no exemption for the dependent child. Current law gives a $15,000 head-of-household deduction plus a $5,000 dependent exemption. It then taxed $54,000 at 5.19% instead of $41,662.50 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its taxable income of $41,687 nearly matches the correct $41,662.50, but only because offsetting errors cancelled out. It subtracted the auto loan interest, used a $12,000 standard deduction, added a nonexistent $3,700 personal exemption, and used a $4,000 dependent exemption, instead of the $15,000 standard deduction plus the $5,000 dependent exemption. The remaining error is that it applied 5.09% instead of Georgia's 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Included the full $12,824 of Social Security dependent benefits in Georgia income, even though Georgia fully subtracts the taxable Social Security in federal AGI. It also used a $12,000 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.12%, instead of a $15,000 head-of-household deduction and a $5,000 dependent exemption at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Subtracted its deduction from $74,824 of gross income that still included the Social Security benefits, despite saying they were excluded. It also used an invented $23,100 standard deduction, producing $51,700 of taxable income instead of $41,662.50. Its $2,361.45 implies a 4.57% rate that matches no Georgia rate."
us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits."
us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit."
us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract."
@@ -7191,120 +7375,124 @@ us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"It skipped
us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"It reasoned that the disability plus $20,814 in mortgage interest and property taxes drove net income to zero, and it awarded the full one-person maximum of $3,492 ($291 x 12). It never checked countable resources. The $31,776 bank balance breaks the $4,500 elderly or disabled asset limit and Texas's $5,000 limit, so the household is ineligible and SNAP is $0."
us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"It used the disability to move straight to the net-income test and let the homeowner shelter costs zero out net income, which produced the maximum one-person allotment of $298 x 12 = $3,576. It never applied the SNAP resource test. The $31,776 bank balance exceeds the $4,500 limit for elderly or disabled households and Texas's $5,000 limit, so the household is ineligible and SNAP is $0."
us,scenario_088,snap,gpt-6-sol,llm_error,asset_resource,False,"It applied the uncapped shelter deduction for disabled households, zeroed out net income, and paid the $298 monthly maximum, for $3,576 a year. It ignored the SNAP resource test. The head's $31,776 in bank assets exceeds the $4,500 elderly or disabled asset limit and Texas's $5,000 limit, so the household is ineligible and SNAP is $0."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly derived AGI $106,200, the $32,200 standard deduction, the $9,568 QBI deduction and 2026 brackets for $7,236. It never applied the $3,467.43 qualified overtime deduction for the spouse's FLSA overtime premium, which lowers taxable income to $60,964 and tax to $6,820. It then discarded its own $7,236 for an unexplained $8,225."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It omitted the $3,467.43 qualified overtime deduction. It also took a ~$1,200 self-employed health insurance deduction, which is barred because both spouses have employer-sponsored coverage, and dropped the head's $252 spousal IRA deduction. That left taxable income at $63,722 instead of $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented ~$57,893 of head wages from weekly hours, although the head has no listed wages (unlisted = 0). It used the stale $29,200 standard deduction and never applied the $9,568 QBI or $3,467 overtime deductions, so its taxable income was ~$127,857 versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It got AGI $106,200 right but used the 2025 $31,500 standard deduction instead of 2026's $32,200 and a $24,150 bracket edge instead of $24,800. It omitted the $3,467 qualified overtime deduction, then inflated its own $7,377 result to $9,089 with unexplained 'adjustments'."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted both the $9,568 QBI deduction and the $3,467 overtime deduction, and used a $31,500 standard deduction, giving taxable income of $74,700. It then added the $4,286 SE tax and other amounts to income tax, although SE tax is a separate liability excluded from this output."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its bracket computation (~$7,150) left out the $3,467 qualified overtime deduction. It then abandoned that figure for $12,850 by asserting 'higher effective rates' on self-employment and partnership income, although that income is taxed at the same 10%/12% ordinary brackets."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) even though the head has no wages for an elective deferral to reduce, which understated AGI at $100,797 versus $106,200. It also used a $30,800 standard deduction and inflated 2025 brackets instead of the 2026 $32,200 and $24,800, and omitted the $3,467 overtime deduction. These errors roughly offset to $6,761."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $19,650 partnership/S-corp income as SE earnings (half SE tax $3,532 instead of $2,143) and disallowed the head's $252 spousal IRA. It used a $30,000 standard deduction and never applied the $9,568 QBI or $3,467 overtime deductions, then rounded its $8,054 up to $8,300 arbitrarily."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Every step matched the reference (AGI $106,200, $32,200 standard deduction, $9,568 QBI, 2026 brackets) except that it omitted the $3,467.43 qualified overtime deduction for the spouse's FLSA overtime premium. That left taxable income $3,467 too high and tax $416 too high ($7,236 vs $6,820)."
-us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $3,467 FLSA overtime premium on top of gross wages that already include it, excluded the $7,789 ESI premiums from wages, and claimed a $2,400 SE health insurance deduction that employer coverage bars. It then used the 2025 $30,000 standard deduction and 2025 brackets, and omitted both the $9,568 QBI and $3,467 overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It misstated spouse wages net of 401(k) as $57,735 instead of $58,935 and used the 2024 $29,200 standard deduction instead of 2026's $32,200. It computed QBI on the full $49,985 without subtracting the $2,143 half-SE-tax deduction, and omitted the $3,467 qualified overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a post-TCJA-sunset regime (personal exemptions, pre-TCJA standard deduction and brackets), although the OBBBA made the TCJA structure permanent with a $32,200 MFJ standard deduction and 10%/12% brackets for 2026. It also excluded the $7,789 ESI premiums from wages (AGI $98,411 vs $106,200) and omitted the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI $106,199.90 correctly but omitted both the $9,568 Section 199A QBI deduction and the $3,467 qualified overtime deduction. It used the $31,500 standard deduction instead of $32,200, leaving taxable income at $74,700 versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 ($16,000 standard deduction, $10,100 personal exemptions, 15% bracket), but the OBBBA kept the $32,200 standard deduction and 10%/12% brackets. It also subtracted the $7,789 ESI premiums from wages and omitted the $9,568 QBI and $3,467 overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its reasoning subtracts only the standard deduction and retirement contributions and never applies the $9,568 QBI deduction or the $3,467 overtime deduction. Its $11,090 corresponds to taxable income near $96,500 under 2026 brackets, about $35,500 above the correct $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a TCJA sunset ($16,600 standard deduction plus $10,600 personal exemptions, pre-TCJA brackets), whereas 2026 law under the OBBBA uses a $32,200 standard deduction, no exemptions and 10%/12% brackets. It also excluded the $7,789 ESI premiums from wages and omitted the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a TCJA-sunset structure ($15,300 standard deduction, $10,100 exemptions, 15% bracket above $18,650) instead of the permanent OBBBA $32,200 standard deduction and 10%/12% brackets. It also subtracted the $7,789 ESI premiums from wages and applied neither the $9,568 QBI nor the $3,467 overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $15,622 exceeds the ~$14,900 that 2026 brackets impose on the household's entire $115,866 of gross receipts with zero deductions, so it both skipped the $32,200 standard deduction, $9,568 QBI deduction and $3,467 overtime deduction and imputed extra income or bundled SE tax. The correct derivation yields $60,964 taxable and $6,819.71 tax."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded the spouse's $7,789 ESI premiums from wages, understating AGI at $98,410.90 versus $106,200.13. It then used a $30,800 standard deduction and the 2024 $23,200 bracket edge instead of $32,200 and $24,800, and omitted the $3,467 qualified overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its ~$98,663 AGI reflects subtracting the $7,789 ESI premiums from the spouse's wages and dropping the head's $252 spousal IRA; the correct AGI is $106,200. It also never applied the $3,467 qualified overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its reasoning cites only a standard deduction, omitting the $9,568 QBI deduction and the $3,467 qualified overtime deduction. Its $7,612 corresponds to about $67,600 of taxable income under 2026 brackets versus the correct $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"AGI ($106,199.91) and QBI ($9,568.38) were right. It used a $30,600 standard deduction and a $24,300 10% bracket edge instead of 2026's $32,200 and $24,800, and omitted the $3,467 qualified overtime deduction, giving taxable income of $66,032 versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $3,467 qualified overtime deduction and wrongly denied the head's $252 spousal IRA, which is deductible on joint compensation under IRC 219(c). It also misstated AGI as ~$104,131 and used the $31,500 standard deduction and a $24,600 bracket edge instead of $32,200 and $24,800."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It declared the liability indeterminable and submitted $0, even though the facts fully specify $65,881 wages, $30,335 SE income and $19,650 partnership income for a joint return. Those produce $60,964 of taxable income and $6,819.71 of tax."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its explanation subtracts only standard deductions and retirement contributions and names neither the $9,568.44 QBI deduction nor the $3,467.43 overtime deduction. Its $6,828 is $8.29 above the exact $6,819.71 (about $69 of excess taxable income), the result of approximation rather than the exact deduction stack."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction but subtracted the $7,789 ESI premiums as 'pretax health' deductions, dropped the head's $252 spousal IRA, and omitted the $9,568 QBI deduction entirely. That yields taxable income of $62,996 versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its deduction list (retirement, half SE tax, standard deduction, QBI) omits the $3,467 qualified overtime deduction for the spouse's FLSA overtime premium. Its $7,530 corresponds to about $66,900 of taxable income versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $7,789 employer-sponsored insurance premiums as a pre-tax Section 125 wage exclusion, but nothing marks them as salary reductions; the head carries identical premiums with no wages. That understated AGI at $98,410.90 instead of $106,200.13. With the correct $32,200, $3,467 and $9,568 deductions, this single error cut tax by $934.65."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $7,789 employer-sponsored insurance premiums as 'eligible pretax wage deductions', lowering AGI to $98,410.90. Employment income is only net of the $6,946 traditional 401(k), so AGI is $106,200.13. Its standard, overtime and QBI deductions were all correct, so this one exclusion accounts for the entire $934.65 shortfall."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated combined income as ~$120,000, above the $108,920 gross income after the spouse's 401(k). It applied only a standard deduction, omitting the $9,568 QBI and $3,467 overtime deductions, and overstated tax by ~$5,600."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"AGI $106,200 was right, but it applied a TCJA-sunset regime ($16,637 standard deduction, $10,611 exemptions, 15% bracket) instead of the OBBBA-permanent $32,200 standard deduction and 10%/12% brackets. It also omitted the $9,568 QBI and $3,467 overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed AGI $106,200 correctly but assumed the TCJA expired ($16,600 standard deduction plus $10,600 exemptions, 15% bracket), whereas 2026 law uses a $32,200 standard deduction with 10%/12% brackets. It also skipped the $9,568 QBI and $3,467 overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly reached AGI $106,200 but applied pre-TCJA 2026 rules ($16,600 standard deduction, $10,600 personal exemptions, 15% bracket) that the OBBBA repealed. It omitted the $9,568 QBI and $3,467 overtime deductions, yielding $79,000 taxable versus $60,964."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $3,467 overtime premium on top of wages that already include it and subtracted the head's $5,403 401(k) with no head wages. It claimed a $600 SE health insurance deduction that employer coverage bars, used a $30,660 standard deduction, and applied neither the QBI nor the overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"AGI and the $32,200 standard deduction were right. It computed QBI as 20% of $49,985 without reducing it by the $2,143 deductible half of SE tax (correct QBI is $47,842.18), omitted the $3,467 qualified overtime deduction, and used a $24,500 bracket edge instead of $24,800."
-us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It double-counted the $3,467 overtime premium on top of gross wages and subtracted the head's $5,403 401(k) though the head has no wages. It used a $29,900 standard deduction and 2025 brackets, and applied neither the $9,568 QBI nor the $3,467 overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reached essentially the correct taxable income ($60,964.52) but used $24,400 as the top of the 2026 MFJ 10% bracket instead of $24,800. That moved $400 from 10% to 12% and overstated tax by exactly $8.00."
-us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $5,403 401(k) despite no head wages and treated the partnership income as SE earnings (half SE tax $3,531 vs $2,143). It used QBI of 20% of the unreduced $49,985, a $30,000 standard deduction and 2025 brackets, and omitted the $3,467 overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) though the head has no wages, understating AGI at $100,797 versus $106,200. It also computed QBI on $49,985 without subtracting the half-SE-tax deduction ($9,997 vs $9,568) and omitted the $3,467 qualified overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $5,403 401(k) with no head wages, used a $32,300 standard deduction, and omitted both the $9,568 QBI and $3,467 overtime deductions. It then applied an internally inconsistent bracket base ($2,783.50 over a $23,200 threshold)."
-us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It invented $132,619 of head wages from the spouse's $27 hourly rate, although the head has no listed wages, and inflated the half-SE-tax deduction to $14,180. It subtracted an $8,000 child tax credit for nonexistent children and a saver's credit, and omitted the QBI and overtime deductions."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Got AGI ($106,200), the $32,200 standard deduction and the $9,568 QBI deduction right, but left out the $3,467 OBBBA qualified overtime deduction, which brought it to $7,236. It then replaced its own figure with an unexplained $8,225."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Took an invented ~$1,200 SE health insurance deduction even though the spouse has employer coverage. It also dropped the head's $252 spousal IRA deduction and understated QBI at ~$9,330. On top of that, it left out the $3,467 qualified overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Made up $57,893 of head wages that were not listed, which pushed gross income to $173,759. It also used the outdated $29,200 standard deduction and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"Gave a rounded $6,800 with no calculation behind it. The correct chain is AGI $106,200.13 minus the $32,200 standard deduction, the $9,568.44 QBI deduction and the $3,467.43 overtime deduction, giving $60,964.27 taxable at 10%/12% = $6,819.71. The submitted number is a round estimate, not a bracket computation."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Got AGI right at $106,200, but used the 2025 $31,500 standard deduction instead of $32,200, approximated QBI at $9,200 instead of $9,568, and left out the $3,467 overtime deduction. It then inflated its own $7,377 to $9,089 with unspecified 'adjustments'."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Got AGI right at $106,200, but skipped both the $9,568 QBI deduction and the $3,467 overtime deduction. It then added the $4,286 SE tax and other amounts into income tax, even though SE tax is not part of income tax before refundable credits."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its own bracket calculation came to about $7,150 without the $3,467 overtime deduction. It then submitted $12,850 based on 'higher effective rates', which no bracket supports, so it effectively added non-income-tax amounts."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Deducted the head's $5,403 traditional 401(k) even though the head has no wages for elective deferrals to come out of. It also used an extrapolated $30,800 standard deduction and $24,517 bracket instead of the OBBBA $32,200 and $24,800, and left out the $3,467 overtime deduction. Its errors happened to roughly cancel out."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Treated the $19,650 partnership/S-corp income as SE earnings, which overstated the half-SE-tax deduction at $3,532. It also dropped the head's spousal IRA, used a $30,000 standard deduction, and skipped both the QBI deduction and the overtime deduction before rounding up to $8,300."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Got AGI ($106,200), the standard deduction ($32,200), QBI ($9,568) and the brackets right, but left out the OBBBA qualified overtime deduction for the spouse's $3,467 FLSA overtime premium. That missing deduction is the entire $416 gap (12% × $3,467)."
+us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Added the $3,467 overtime premium on top of wages that already include it, and excluded the $7,789 ESI premium from wages. It also claimed a $2,400 SE health insurance deduction, used a $30,000 standard deduction and 2025 brackets, and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Miscalculated spouse wages net of the 401(k) as $57,735 instead of $58,935, and used the 2024 $29,200 standard deduction. It took QBI as 20% of $49,985 without subtracting the half-SE-tax deduction, applied 2025 brackets, and left out the $3,467 overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA expired in 2026 and used personal exemptions with pre-TCJA rates, but OBBBA made the TCJA rates and the $32,200 standard deduction permanent. It also excluded the ESI premium from wages and skipped the QBI deduction and overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $106,200, but used a $31,500 standard deduction and a $24,400 bracket edge. It also left out both the $9,568 QBI deduction and the $3,467 overtime deduction, which overstated taxable income by about $13,700."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied a TCJA sunset that did not happen: a $16,000 standard deduction, personal exemptions and a 15% bracket, instead of OBBBA's $32,200 deduction and 10%/12% brackets. It also subtracted the $7,789 ESI premium from wages and skipped the QBI deduction and overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Applied only the standard deduction and retirement contributions, with no QBI deduction and no overtime deduction. Even AGI $106,200 less $32,200 gives just $8,384 of tax, so its $11,090 also overstates income or rates beyond those omissions."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Assumed the TCJA sunset, using a $16,600 standard deduction, $10,600 of exemptions and pre-TCJA brackets, instead of the OBBBA-permanent $32,200 deduction and 10%/12% brackets. It also excluded the ESI premium from wages and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied pre-TCJA law: a $15,300 standard deduction, personal exemptions, and a 15% bracket above $18,650. It also excluded the $7,789 ESI premium from wages and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave $15,622 with no derivation, more than double the correct $6,819.71. That is more tax than even undeducted gross income at 10%/12% produces, so it ignored the standard, QBI and overtime deductions and overstated income."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Excluded the spouse's $7,789 ESI premium from wages, which dropped AGI to $98,411 instead of $106,200. It also used a $30,800 standard deduction instead of $32,200 and left out the $3,467 qualified overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Started from an AGI of about $98,663, which reflects excluding the $7,789 ESI premium from wages; the correct AGI is $106,200. It did not apply the $32,200 standard deduction together with the $3,467 overtime deduction, so it ended up at $6,647."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Listed only the standard deduction and named no QBI deduction or overtime deduction. Its $7,612 implies about $67,600 of taxable income, roughly $6,600 above the correct $60,964.27, which is consistent with leaving out the $3,467 overtime deduction and understating QBI or the standard deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Got AGI ($106,199.91) and QBI ($9,568.38) right, but used a $30,600 standard deduction and $24,300 bracket instead of OBBBA's $32,200 and $24,800. It also left out the $3,467 qualified overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Disallowed the head's $252 IRA even though a spousal IRA is deductible against the working spouse's compensation, and made an arithmetic error that put AGI at $104,131 instead of $106,200. It also used a $31,500 standard deduction and left out the $3,467 overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"Returned $0, claiming the facts were incomplete, even though the household lists $65,881 of wages, $30,335 of SE income and $19,650 of partnership income. These produce $60,964.27 of taxable income and $6,819.71 of tax."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $6,828 implies about $61,033 of taxable income, roughly $69 above the correct $60,964.27. It approximated the half-SE-tax, $9,568.44 QBI and $3,467.43 overtime deductions instead of computing them exactly."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Applied the overtime deduction but excluded the $7,789 ESI premium from wages, dropped the head's $252 spousal IRA, and left out the $9,568 QBI deduction entirely. Those changes net out to its $62,996 taxable income, which is $2,032 too high."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its list of deductions leaves out the OBBBA qualified overtime deduction for the $3,467 FLSA premium. Its $7,530 implies about $66,900 of taxable income, so it also understated the standard deduction or QBI deduction by about $2,450."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Treated the spouse's $7,789 ESI premium as a pre-tax wage exclusion, which cut AGI to $98,410.90 instead of $106,200.13. That one error lowered taxable income to $53,175.52; all of its other deductions are correct."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $7,789 ESI premium from wages as a pre-tax exclusion, which gave an AGI of $98,410.90 instead of $106,200.13. That understated taxable income by $7,789 even though its standard, overtime and QBI deductions were correct."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Treated income as 'around 120k' and applied only the standard deduction and retirement contributions. It never took the $9,568 QBI deduction or the $3,467 overtime deduction, and its $12,450 also exceeds what 10%/12% brackets produce on AGI minus the standard deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Got AGI right at $106,200, but applied a TCJA sunset (a $16,637 standard deduction, $10,611 of exemptions and a 15% bracket) that OBBBA prevented. As a result it also skipped the QBI deduction and overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Got AGI right at $106,200, but applied pre-TCJA 2026 law with a $16,600 standard deduction, personal exemptions and a 15% bracket, instead of OBBBA's permanent $32,200 deduction and 10%/12% brackets. It also left out the QBI deduction and overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got AGI right at $106,200, but taxed it under a TCJA sunset that did not happen, with a $16,600 standard deduction, $10,600 of exemptions and a 15% rate. It left out the $9,568 QBI deduction and the $3,467 OBBBA overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Added the $3,467 overtime premium on top of wages that already include it, which put gross income at $119,333. It also deducted the head's $5,403 401(k) despite the head having no wages, claimed a $600 SE health insurance deduction, and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Computed QBI as 20% of $49,985 without first subtracting the $2,143 half-SE-tax deduction from SE QBI, which gave $9,997 instead of $9,568. It also used a $24,500 bracket edge and left out the $3,467 qualified overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Added the $3,467 overtime premium on top of wages that already include it, and deducted the head's 401(k) despite the head having no wages. It also used a $29,900 standard deduction and 2025 brackets, and skipped both the QBI deduction and the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Got taxable income essentially right at $60,964.52, but used a $24,400 top for the 10% bracket instead of the 2026 MFJ $24,800, which added $8. It also placed the overtime deduction above the line rather than after AGI."
+us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Deducted the head's $5,403 401(k) despite the head having no wages, and treated the partnership income as SE earnings, which overstated the half-SE-tax deduction at $3,531. It also took QBI on the unreduced $49,985, used a $30,000 standard deduction and 2025 brackets, and left out the overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Deducted the head's $5,403 401(k) despite the head having no wages, which put AGI at $100,797 instead of $106,200. It also took QBI on the unreduced $49,985 ($9,997 instead of $9,568) and left out the $3,467 overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Deducted the head's 401(k) despite the head having no wages, used a $32,300 standard deduction, and skipped both the QBI deduction and the overtime deduction. It also miscalculated the 10% bracket tax as $2,783.50 on $23,200, which should be $2,320."
+us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Invented $132,619 of head wages from the spouse's hourly wage fields and a $14,180 half-SE-tax deduction. It then subtracted an $8,000 child tax credit even though the household has no children."
us,scenario_089,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded a $2,242.80 Additional Child Tax Credit despite the household having no qualifying children. It also fabricated $120,000 of head wages from hours worked and mislabeled $2,400 of supposed excess Additional Medicare withholding as a refundable credit, even though no head wage or hourly rate was listed and tax withholding refunds are not included in federal_refundable_credits."
-us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It made up a $6,200 payroll tax for the head, who has no listed wages, so their FICA wages are $0. It also added the $3,467 overtime premium on top of $65,881 gross wages that already include overtime, and it added a North Carolina state payroll tax that does not exist. The correct total is 7.65% of $65,881, or $5,039.91."
-us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"It used the right base of $65,881 and reached $5,039.89. It then miscalculated Medicare as $955.57 instead of $955.27 and made an unexplained upward 'variance' adjustment to $5,085.44. Nothing in the payroll tax rules supports that adjustment."
-us,scenario_089,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It added the $3,467 overtime premium to gross wages that already include it. It then subtracted the $6,946 traditional 401(k) deferral, even though elective deferrals remain FICA wages under IRC §3121(v)(1)(A). It also subtracted the $7,789 ESI premium, although the facts list no pre-tax salary reduction. The result was a $54,613 base instead of $65,881."
-us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted $1,200 of 'health insurance premiums excluding Medicare Part B' and 'other health insurance premiums' from wages. These are individually paid premiums, not salary reductions, so they do not reduce the FICA base. That left $64,681 instead of the full $65,881."
-us,scenario_089,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax cafeteria-plan exclusion and cut FICA wages to $58,092. The facts list no pre-tax salary reduction, so the full $65,881 is taxable, which gives $5,039.91 instead of $4,444."
-us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $7,789 employer-sponsored insurance premium from the spouse's $65,881 wages as if it were a Section 125 salary reduction. The listed facts do not reduce the payroll wage base, so FICA applies to all $65,881, giving $5,039.91."
-us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It added the $3,467 FLSA overtime premium to $65,881 gross wages that already include overtime, making the base $69,348. It also miscalculated 7.65% of that base, which is $5,305 rather than $5,373. The correct base of $65,881 yields $5,039.91."
-us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax deduction from FICA wages and got $58,092 × 7.65% = $4,444.04. The spouse's full $65,881 gross wage is the payroll tax base because no salary-reduction election is listed."
-us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It labeled the $7,789 ESI premium 'pre-tax' and removed it from FICA wages, leaving $58,092. With no Section 125 election in the facts, Social Security and Medicare apply to the full $65,881, which yields $5,039.91."
-us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted $7,789 of 'pre-tax ESI premiums' from gross wages to get a $58,092 FICA base. The household facts give no pre-tax salary reduction, so the base is the full $65,881 and the tax is $5,039.91."
-us,scenario_089,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied a 'pre-tax ESI deduction' of $7,789 to reach $58,092 in taxable wages. Employee FICA applies to the spouse's full $65,881 gross wage because no cafeteria-plan salary reduction is listed, giving $5,039.91."
-us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It correctly named the spouse's $65,881 wages as the only payroll tax base, which gives 7.65% × $65,881 = $5,039.91. It then submitted $8,107, about $3,067 higher than its own stated base supports. That figure matches a base of about $106,000, which means non-wage income was folded into it."
-us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It applied FICA to 'W-2 wages for both spouses', but the head has no listed wages and therefore owes no employee payroll tax. The $9,711 answer implies a base of about $127,000, far above the spouse's $65,881 of actual wages, which yield $5,039.91."
-us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It applied an 'employee health-premium exclusion' of $7,789 to reach $58,092 in FICA wages. The facts contain no pre-tax salary-reduction election, so payroll tax applies to all $65,881, giving $5,039.91 rather than $4,444."
-us,scenario_089,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as pretax and reduced FICA wages to $58,092. The payroll tax base is the spouse's full $65,881 gross wage, so the correct figure is $5,039.91."
-us,scenario_089,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the traditional 401(k) deferrals in payroll wages but subtracted the $7,789 employer-sponsored insurance premium, leaving $58,092. No pre-tax salary-reduction election is listed, so FICA applies to the full $65,881, which yields $5,039.91."
-us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"It taxed 'spouse wages and overtime' as separate amounts, but the $65,881 gross wage already includes overtime. The $8,120 answer implies a base of about $106,000, so it also pulled in non-wage self-employment or partnership income. The correct base of $65,881 yields $5,039.91."
-us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It added the $3,467 FLSA overtime premium to the $65,881 gross wage, making the base $69,348. The prompt states that gross wages already include overtime pay, so the base is $65,881 and the tax is $5,039.91, not $5,305."
-us,scenario_089,payroll_tax,inkling,llm_error,other,False,"It used the correct $65,881 base but miscalculated 6.2% × $65,881 as $4,083 instead of $4,084.63, and truncated Medicare to $955 instead of $955.28. That arithmetic slip left it at $5,038 rather than $5,039.91."
-us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It correctly computed $5,039.89 of employee FICA on $65,881, then added $2,145.68 as the 'employee portion' of self-employment tax. Self-employment tax is not employee payroll tax and the question explicitly excludes it."
-us,scenario_089,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Its $10,068.71 of Social Security and $1,416.24 of Medicare imply bases of about $162,000 and $97,700. It folded self-employment and partnership income into the employee FICA base, when only the spouse's $65,881 of wages count, yielding $4,084.63 plus $955.28."
-us,scenario_089,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"It identified the correct base ($30,335 of self-employment income, partnership/S-corp income excluded) and the correct 92.35% × 15.3% formula, but mis-multiplied: 30,335 × 0.9235 = 28,014.37, not the 27,998.80 it wrote. It then discarded even its own $4,285.34 result and submitted $4,275.23 under the label ""standard rounding,"" whereas correct execution of its stated method gives the reference $4,286.24."
-us,scenario_089,self_employment_tax,claude-sonnet-4.6,llm_error,other,False,"Its rule application was exactly right — partnership/S-corp income excluded from net earnings from self-employment, 92.35% × 15.3% on the $30,335 — but the arithmetic slipped: 30,335 × 0.9235 = 28,014.37, not the 28,024.32 it used. That $9.95 overstatement of net earnings carried through the 15.3% rate to $4,287.72 instead of $4,286.24."
-us,scenario_089,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"It folded the $19,650 of partnership/S-corp income into the SECA base for a $49,985 total; PolicyEngine's self-employment tax draws only on the self-employment income field of $30,335, and partnership/S-corp income flows to taxable income without entering net earnings from self-employment. It compounded that by computing $7,062.66 from the inflated base and then submitting an unexplained $6,355 that its own arithmetic never produced."
-us,scenario_089,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"It named the correct computation — 15.3% of 92.35% of the $30,335 self-employment income — but executed it wrong. That product is $4,286.24, while its $4,357 corresponds to a net-earnings base of $28,477, about $463 above the 92.35% × $30,335 = $28,014.37 its own sentence specifies."
-us,scenario_089,self_employment_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"It taxed the $19,650 of partnership/S-corp income alongside the $30,335 of self-employment income, but only the self-employment income field enters net earnings from self-employment. The $19,650 adds roughly $2,777 of SECA tax on top of the $4,286.24 reference, which is what puts its $7,117 in that range."
-us,scenario_089,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It stated the right base — the $30,335 self-employment income with partnership/S-corp income excluded — then never applied the 92.35% net-earnings reduction and used a base larger than even the gross figure: $5,114 is 15.3% of $33,425, versus the $28,014.37 base its own reasoning implies. Applying 15.3% to 92.35% of $30,335 yields the reference $4,286.24."
-us,scenario_089,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"It recited the correct method — 15.3% on 92.35% of the $30,335 — which produces $4,286.24, then submitted $3,677, a figure equal to 15.3% of only $24,032, i.e. 79% of the self-employment income rather than 92.35%. Its aside that wages ""exceed the threshold only marginally"" is also wrong: the Additional Medicare Tax threshold for joint filers is $250,000, far above this household's $69,348 of wages."
-us,scenario_089,self_employment_tax,grok-4.3,llm_error,other,False,"Its $4,280 is a truncated approximation of 0.153 × 0.9235 × $30,335 = $4,286.24, losing $6.24 by rounding the net-earnings step instead of carrying cents. Its stated inclusion of partnership income contradicts its own number, since adding the $19,650 to the SECA base would have produced about $7,063."
-us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for self_employment_tax, so the failure is a missing submission rather than a substantive computation error. The required derivation is 92.35% of the spouse's $30,335 self-employment income taxed at 15.3%, with the $19,650 partnership/S-corp income excluded from the SECA base, giving $4,286.24."
-us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"It combined the $19,650 partnership/S-corp income with the $30,335 self-employment income into a single $49,985 SECA base. Partnership/S-corp income is outside net earnings from self-employment in PolicyEngine, so its $7,063 overstates the $4,286.24 reference by precisely the 14.13% effective SECA tax on that $19,650."
-us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It explicitly weighed whether partnership/S-corp income belongs in net earnings from self-employment and resolved it the wrong way, taxing the full $49,985. The SECA base here is the $30,335 self-employment income alone, which at 92.35% × 15.3% gives $4,286.24 rather than its $7,062.66."
-us,scenario_089,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It made two errors: it inflated the base to ~$50,000 by adding the $19,650 of partnership/S-corp income, which does not enter net earnings from self-employment, and it then subtracted a fabricated $668.78 ""offset"" for Social Security tax already paid on wages. Wage-base coordination reduces the SE Social Security base only when wages exceed the annual Social Security cap, and this household's $69,348 of wages are far below it; on the correct $30,335 base with no offset the tax is $4,286.24."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached NC taxable income of $80,952 and tax of $3,230, which already contained an error: it disallowed Head's $252 traditional IRA deduction, but on a joint return that deduction is allowed as a spousal IRA under IRC §219(c). It then submitted $4,022, a number no step of its work produces. The correct base is AGI of $106,200.13 minus the $25,500 joint standard deduction, taxed at 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the right 3.99% rate and $25,500 standard deduction but started from a federal AGI of $105,252, which is $948 below the correct $106,200.13. Correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus $30,335 self-employment and $19,650 pass-through income, minus about $2,143 half-SE-tax and $577 of IRA deductions. As a result NC taxable income was understated at $79,752 instead of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It showed no standard-deduction or rate step, and its $4,532 is roughly 4.25% of the full ~$106,200 AGI. That matches taxing AGI without subtracting North Carolina's $25,500 joint standard deduction and using an outdated rate instead of the 2026 rate of 3.99%. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It deducted Head's $5,403 traditional 401(k) deferral even though Head has no wages; 401(k) deferrals can only come out of pay. That gave AGI of $100,797 and tax of $3,004, but it then submitted $4,710, which none of its calculations support. Excluding only the spouse's $6,946 deferral gives AGI of $106,200.13, NC taxable income of $80,700.13, and tax of $3,219.94 at 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning followed the correct path: it excluded only the spouse's 401(k), took the $577 IRA and ~$2,144 half-SE-tax adjustments, reached AGI of ~$106,199, subtracted $25,500, and applied 3.99% to get $3,220. It then dropped that result for an invented ~$132,500 income base, larger than the household's entire $115,866 gross income, and submitted $5,290."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stated NC taxable income of about $81,000 (AGI ~$106,500 minus $25,500) but then multiplied 3.99% by $87,100, inflating its own base by about $6,100 with no explanation. Applying 3.99% to the correct $80,700.13 gives $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a made-up $30,000 joint standard deduction (North Carolina's is $25,500 under G.S. 105-153.5) and a 4.25% rate instead of the 3.99% rate set for 2026 under G.S. 105-153.7. Its AGI of $106,126 was close, so the two errors partly cancelled out to $3,235."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It counted only 92.35% of self-employment income in AGI, but the 0.9235 factor applies only to the self-employment tax base. It also treated the $19,650 partnership/S-corp income as subject to self-employment tax and used 4.25% instead of 3.99%. Finally it submitted $3,800, which is not a rounding of its own $2,979 result."
-us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $99,477, which is $6,723 below the correct $106,200.13; correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus self-employment and pass-through income, minus half-SE-tax and $577 of IRA deductions. It also used a $27,500 joint standard deduction instead of North Carolina's $25,500, so NC taxable income came out at $71,977 instead of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $104,999.90 is $1,200 below the correct $106,200.13, which matches deducting the two spouses' $600 'other health insurance premiums' as a self-employed health insurance deduction. IRC §162(l)(2)(B) bars that deduction because both spouses are eligible for employer-sponsored coverage. Without it, ($106,200.13 − $25,500) × 3.99% = $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It removed the spouse's $7,789 employer-sponsored insurance premium from wages, giving AGI of $98,411 instead of $106,200.13. It also used a $30,600 joint standard deduction instead of North Carolina's $25,500 and a 4.0% rate instead of the 2026 rate of 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from AGI of $98,411, which is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium. The facts give gross wages with no pre-tax payroll deduction for that premium, so AGI stays $106,200.13 and NC taxable income is $80,700.13, not $72,911."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $4,410 equals 3.99% of about $110,500, which is more than AGI, so it never subtracted North Carolina's $25,500 joint standard deduction from an income base close to gross income. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $98,411, which removes the spouse's $7,789 employer-sponsored insurance premium from gross wages; the facts give no pre-tax payroll deduction for that premium. The correct AGI of $106,200.13 minus the $25,500 standard deduction gives $80,700.13 of taxable income, not $72,911."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from AGI of $98,411 by wrongly removing the spouse's $7,789 employer-sponsored insurance premium from wages. It also subtracted a $29,250 standard deduction instead of North Carolina's $25,500 joint amount, producing taxable income of $69,161 instead of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It showed no calculation. $6,787 is more than 3.99% of the household's entire $115,866 gross income ($4,623), so it did not apply North Carolina's 2026 flat 3.99% rate to AGI minus the $25,500 joint standard deduction. That calculation gives $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its NC AGI of $98,410.90 is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, which it treated as a pre-tax wage deduction the facts do not support. NC taxable income is $80,700.13, not $72,910.90."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $2,919.20 is 3.99% of $73,163, which implies AGI of $98,663. That equals the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, with Head's $252 IRA deduction added back. So it wrongly treated the insurance premium as a pre-tax wage deduction and disallowed Head's spousal IRA deduction, which IRC §219(c) permits on a joint return."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $3,514 is 3.99% of about $88,070 of taxable income, roughly $7,370 more than the correct $80,700.13. The overstatement is about the size of the spouse's $6,946 traditional 401(k) deferral plus IRA deductions, which it left in income."
-us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It counted self-employment income in AGI as $28,014.47, the $30,335 multiplied by 0.9235, but that factor applies only to the self-employment tax base. AGI includes the full $30,335 minus the $2,143 half-SE-tax deduction, which gives $106,200.13 instead of $103,879.36."
-us,scenario_089,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($63,098) instead of federal AGI, which is North Carolina's actual starting point under G.S. 105-153.5. It then subtracted a $21,500 standard deduction instead of the $25,500 joint amount. The correct base is $106,200.13 − $25,500 = $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It returned $0 even though federal AGI of $106,200.13 leaves $80,700.13 of NC taxable income after the $25,500 joint standard deduction. North Carolina has no nonrefundable credit for this childless couple that offsets the resulting $3,219.94 of tax at 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $3,055 is 3.99% of about $76,570 of taxable income, roughly $4,130 below the correct $80,700.13. It subtracted too much for retirement contributions: only the spouse's $6,946 traditional 401(k) deferral (Head has no wages) and the couple's $577 of IRA contributions reduce AGI."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It removed $7,789 of 'pretax ESI premiums' from the spouse's wages, lowering AGI to $98,411. The facts give gross wages with no pre-tax premium deduction, so AGI is $106,200.13 and NC taxable income is $80,700.13, not $72,911."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its NC taxable income of $69,696 is $11,004 below the correct $80,700.13 (AGI of $106,200.13 minus $25,500). It subtracted items that do not reduce AGI in this household, such as the spouse's $7,789 employer-sponsored insurance premium and deferrals beyond the spouse's own $6,946 401(k) contribution."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,230 is 3.99% of $80,952, which implies AGI of $106,452: the correct $106,200.13 with Head's $252 traditional IRA deduction taken out. Head has no earnings, but on a joint return the spousal-IRA rule in IRC §219(c) lets Head deduct against the spouse's earnings, so the full $577 IRA deduction applies."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $98,410.90 is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, which it treated as a pre-tax reduction of the stated gross wages without any basis in the facts. NC taxable income is $80,700.13, not $72,910.90."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $98,410.90 removes the spouse's $7,789 employer-sponsored insurance premium from wages as if it were a pre-tax payroll deduction, which the facts do not state. The correct AGI of $106,200.13 minus the $25,500 standard deduction gives $80,700.13 taxed at 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $3,120 is 3.99% of about $78,200 of taxable income, roughly $2,500 below the correct $80,700.13. So it understated the correct $106,200.13 AGI by over-reducing self-employment or retirement items before subtracting the $25,500 standard deduction."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got AGI ($106,200) and taxable income ($80,700) right but applied 4.25%, North Carolina's 2025 rate, instead of the 3.99% rate for 2026 under G.S. 105-153.7."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got the $106,200 AGI and $80,700 taxable income right but taxed them at the 2025 rate of 4.25% instead of North Carolina's 2026 rate of 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used AGI of $103,664, which is $2,536 below the correct $106,200.13, and a 4% rate instead of the 2026 rate of 3.99%. Correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus the full $30,335 self-employment and $19,650 pass-through income, minus $2,143 half-SE-tax and $577 of IRA deductions, which leaves taxable income of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It started from the correct ~$106,200 AGI but subtracted a ~$26,700 joint standard deduction instead of North Carolina's statutory $25,500 (G.S. 105-153.5(a)(1)). That understated taxable income at ~$79,500 instead of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $102,732.90 is the correct $106,200.13 minus the spouse's $3,467 FLSA overtime premium. The federal qualified-overtime deduction under IRC §225 is taken after AGI and does not reduce it, so it never reaches North Carolina's AGI starting point."
-us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied 4.5%, North Carolina's 2024 rate, instead of the 3.99% rate for 2026. Its AGI of $99,409 is also $6,791 below the correct $106,200.13."
-us,scenario_089,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI of $100,797 subtracts Head's $5,403 traditional 401(k) deferral even though Head has no wages, and 401(k) deferrals can only come out of pay. Excluding only the spouse's $6,946 deferral gives AGI of $106,200.13 and taxable income of $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It applied 4.5% instead of the 2026 rate of 3.99%. It also added the $3,467 FLSA overtime premium on top of gross wages that already include it, computed self-employment tax on the $19,650 partnership/S-corp income as if it were self-employment income, and deducted Head's $5,403 401(k) deferral even though Head has no wages."
-us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used an invented AGI of $248,485, more than double the household's $115,866 gross income, along with a $32,200 standard deduction and a 4.5% rate. It also made up a credit for taxes paid to other states, although no fact places the partnership income outside North Carolina. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94."
+us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It made up about $6,200 of payroll tax for the head, who has no wages listed (the prompt says to treat unlisted amounts as 0). It also added the $3,467 overtime premium on top of the $65,881 gross wages, which already include overtime, and invented an NC employee payroll tax that does not exist. The correct base is just the spouse's $65,881, giving $5,039.91."
+us,scenario_089,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"It charged FICA on wages for both spouses even though the head has no wages. It also used $69,348 by adding the overtime premium to gross wages that already include overtime. Only the spouse's $65,881 is taxable, which comes to $5,039.91, not $10,400."
+us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"It used the right base, $65,881 × 7.65%, and got $5,039.89. Then it miscalculated the Medicare piece as $955.57 instead of $955.27 and, with no stated reason, raised the total to $5,085.44 'accounting for slight variance'. That moved it away from its own correct figure."
+us,scenario_089,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It added the $3,467 overtime premium to gross wages that already include overtime. It then subtracted both the $6,946 traditional 401(k) deferral and the $7,789 ESI premium, ending with $54,613. Elective 401(k) deferrals stay in FICA wages under IRC §3121(v)(1)(A), and PolicyEngine does not reduce payroll wages by the ESI premium, so the base is $65,881."
+us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted $1,200 of 'health insurance premiums' from the spouse's wages. Those premiums are not a payroll-tax exclusion, so the FICA base stays at the full $65,881 and the tax is $5,039.91, not $4,948.10."
+us,scenario_089,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax cafeteria-plan exclusion and taxed $58,092. PolicyEngine does not reduce payroll-tax wages by this premium, so all $65,881 is taxed at 7.65%."
+us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $7,789 ESI premium and taxed $58,092. In PolicyEngine the ESI premium does not reduce FICA wages, so 7.65% applies to the full $65,881, which gives $5,039.91."
+us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It used $69,348 by adding the $3,467 FLSA overtime premium to the $65,881 gross wages. The prompt says gross wages already include overtime, so this counts overtime twice. The base is $65,881."
+us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax deduction and taxed $58,092. PolicyEngine's payroll-tax wage base is the full $65,881 of gross wages, with no ESI premium exclusion."
+us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted $7,789 of ESI premiums and taxed $58,092. Those premiums do not reduce FICA wages in PolicyEngine, so 7.65% of $65,881 gives $5,039.91."
+us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It took the $7,789 ESI premium off FICA wages, leaving $58,092. The correct base is the spouse's full $65,881 of gross wages, because PolicyEngine does not apply that premium as a payroll-tax exclusion."
+us,scenario_089,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied a pre-tax ESI deduction to get $58,092 of FICA wages. PolicyEngine taxes the full $65,881 of gross wages for Social Security and Medicare without subtracting the premium."
+us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It correctly said the base is the spouse's $65,881 of wages, then reported $8,107. That number is 7.65% of about $106,000, so it pulled in non-wage income. Applying its own stated rule gives 7.65% × $65,881 = $5,039.91."
+us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It applied FICA to 'W-2 wages for both spouses' even though the head has no wages. $9,711 is 7.65% of about $127,000, which can only come from invented head wages or self-employment and partnership income. Only the spouse's $65,881 counts."
+us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It treated the $7,789 ESI premium as an employee health-premium exclusion and taxed $58,092. PolicyEngine's payroll-tax base is the full $65,881 of gross wages, so the result is $5,039.91."
+us,scenario_089,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It subtracted the $7,789 'pretax' employer-sponsored insurance premium and taxed $58,092. PolicyEngine does not reduce FICA wages by the ESI premium, so 7.65% applies to $65,881."
+us,scenario_089,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the 401(k) deferrals in the base but subtracted the $7,789 ESI premium to get $58,092. The full $65,881 of gross wages is the payroll-tax base, which gives $5,039.91."
+us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"It taxed 'spouse wages and overtime', which counts the $3,467 overtime premium twice because it is already in gross wages. Its $8,120 is 7.65% of about $106,000, so it also included self-employment or partnership income in the base. The correct base is $65,881."
+us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It used $69,348 by adding the $3,467 FLSA overtime premium to gross wages that the prompt says already include overtime. Taxing only the $65,881 gives $5,039.91."
+us,scenario_089,payroll_tax,inkling,llm_error,other,False,"The base ($65,881) and rates were right, but the arithmetic was wrong. It wrote 6.2% × $65,881 as $4,083 when it is $4,084.62, and rounded Medicare down to $955. The correct total is $5,039.91, not $5,038."
+us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It correctly found $5,039.89 of employee FICA on the $65,881 of wages. Then it added half of self-employment tax ($2,145.68), even though the requested payroll_tax output explicitly excludes self-employment tax."
+us,scenario_089,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Its $10,068.71 of Social Security tax implies a base of about $162,000, and its $1,416.24 of Medicare tax implies about $98,000. Both are far above the $65,881 wage total, so it counted self-employment and partnership income or other non-wage amounts as FICA wages. 7.65% of $65,881 gives $5,039.91."
+us,scenario_089,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"The model used the right base and rates but multiplied wrong: $30,335 × 0.9235 is $28,014.37, not $27,998.80. It then swapped its own computed $4,285.34 for an unexplained $4,275.23 labeled as 'rounding'. The correct product, $28,014 × 15.3%, is about $4,286."
+us,scenario_089,self_employment_tax,claude-haiku-5.5,llm_error,other,False,"The method was right: 92.35% of the $30,335 SE income times 15.3%. But the model submitted a rough $4,290 instead of computing the exact $28,014 × 0.153 ≈ $4,286. The $3.76 gap comes entirely from that loose rounding."
+us,scenario_089,self_employment_tax,claude-sonnet-4.6,llm_error,other,False,"The model correctly left out the partnership/S-corp income and applied 0.9235 × 15.3%. But it multiplied $30,335 × 0.9235 as $28,024.32 instead of $28,014.37. That $10 error in the base inflated the tax to $4,287.72 instead of about $4,286."
+us,scenario_089,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model added the $19,650 of partnership/S-corp income to the $30,335 of SE income to get a $49,985 base. That partnership/S-corp income is not part of net earnings from self-employment here. It then submitted $6,355, which doesn't match its own $7,063 calculation. Using only the $30,335 of SE income gives about $4,286."
+us,scenario_089,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model described the correct formula, 15.3% × 92.35% × $30,335. But its answer of $4,357 implies a base of about $28,477 instead of $28,014. Doing the arithmetic correctly gives about $4,286, so the $71 overstatement is a calculation error."
+us,scenario_089,self_employment_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model explicitly taxed the spouse's partnership/S-corp income along with the SE income. That put the $19,650 into the SE base, which is limited to the $30,335 of self-employment income. Its $7,117 is even above the $7,063 that the combined base would give. The correct base gives about $4,286."
+us,scenario_089,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly said only the $30,335 of SE income is taxed, but it never applied the 92.35% factor and 15.3% rate correctly. Its $5,114 is $828 too high. The correct result is $30,335 × 0.9235 × 0.153 ≈ $4,286."
+us,scenario_089,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model named the correct 92.35% × 15.3% method on $30,335 but submitted $3,677, which implies a base of only about $24,033 instead of $28,014. It also brought in Additional Medicare Tax, which has nothing to do with the SE tax here. The correct calculation gives about $4,286."
+us,scenario_089,self_employment_tax,grok-4.3,llm_error,other,False,"The model said it included partnership income 'after deductions', but that income is not part of the SE tax base. Its $4,280 is just a rounded estimate of the SE-only figure, not the exact $28,014 × 15.3% ≈ $4,286."
+us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for self_employment_tax, so there was no answer to score. The correct calculation is $30,335 × 0.9235 × 15.3% ≈ $4,286."
+us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model combined the $19,650 of partnership/S-corp income with the $30,335 of SE income, for a $49,985 base, and taxed $46,161 of net earnings. Only the $30,335 belongs in the SE base. Counting the $19,650 inflated the tax from about $4,286 to $7,063."
+us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model noticed that S-corp income generally isn't subject to SE tax, but then treated all $19,650 as partnership earnings and added it to the base. Only the $30,335 of SE income counts. Applying 92.35% × 15.3% to that alone gives about $4,286, not $7,062.66."
+us,scenario_089,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model rounded the SE income plus the partnership/S-corp income to a combined $50,000 base, but only the $30,335 of SE income counts. It then subtracted a made-up $668.78 'offset' for employee Social Security tax paid on wages. Wages only reduce the 12.4% base once wages plus SE earnings exceed the 2026 Social Security wage base. The spouse's roughly $66k in wages plus $28k in net SE earnings is far below that, so no offset applies."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It wrongly disallowed the head's $252 traditional IRA. That contribution is deductible under the spousal-IRA rule using the spouse's compensation, and dropping it gave $80,952 of taxable income and $3,230 of tax instead of $80,700 and $3,219.94. It then submitted $4,022, which matches none of its own arithmetic."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The rate (3.99%) and standard deduction ($25,500) are right, but it understated federal AGI as $105,252 instead of $106,200. The extra ~$948 of deductions has no basis in the facts, so taxable income came out at $79,752 instead of $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It listed the right income parts but never computed NC taxable income. $4,532 at 3.99% implies about $113,600 of taxable income, which means the $25,500 MFJ standard deduction was never effectively subtracted from the $106,200 AGI."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted unspecified 'retirement/health adjustments' beyond what federal AGI allows. $2,700 at 3.99% implies only ~$67,700 of taxable income, so it removed health premium or medical inputs that do not reduce AGI. The correct base is $106,200 − $25,500 = $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $5,403 traditional 401(k), giving an AGI of $100,797. The head has no wages, so no elective deferral can reduce income. It then dropped its own $3,004 result and submitted an unsupported $4,710."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It correctly computed AGI of $106,199, taxable income of $80,699 and tax of $3,220 at 3.99%. It then threw that result away for an invented ~$132,500 'full income base' and submitted $5,290."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stated AGI of ~$106,500 and taxable income of ~$81,000, which would give about $3,232. It then multiplied 3.99% by an unexplained $87,100, an arithmetic inconsistency that pushed the tax up to $3,475."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a 4.25% rate and a $30,000 MFJ standard deduction. The correct 2026 parameters are 3.99% and $25,500. It also miscomputed SE tax (30,335 × 0.9235 is $28,014, not $28,974). Its $3,235 is the result of these errors offsetting each other, not of the correct $80,700 × 3.99% calculation."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It wrongly applied the 92.35% SE-tax factor to the income itself, cutting the pass-through and SE income included in AGI. It also treated the partnership/S-corp income as SE income and used a 4.25% rate instead of 3.99%. It then rounded its own $2,979 up to an unsupported $3,800."
+us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used AGI of $99,477 instead of $106,200, an unexplained understatement of about $6,700. It also used a $27,500 MFJ standard deduction instead of NC's $25,500, so taxable income was $71,977 instead of $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $104,999.90 is exactly $1,200 below the correct $106,199.90. It subtracted the $600 + $600 'other health insurance premiums' as a self-employed health insurance deduction, which is not allowed because both spouses have employer-sponsored coverage."
+us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used AGI of $98,411, which subtracts the spouse's $7,789 ESI premium input as a pretax wage reduction even though it does not reduce AGI. It also used a $30,600 standard deduction instead of $25,500 and a 4.0% rate instead of 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $98,411 AGI is exactly $7,789 below $106,200, meaning it subtracted the spouse's employer-sponsored insurance premium from wages as a pretax deduction. That input does not reduce AGI. With the correct $106,200 AGI, taxable income is $80,700, not $72,911."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $4,410 at 3.99% implies about $110,500 of taxable income, which is above the $106,200 AGI, so the $25,500 MFJ standard deduction was never subtracted."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $98,411, which wrongly subtracts the spouse's $7,789 ESI premium input from wages. The correct AGI is $106,200, giving $80,700 of NC taxable income instead of $72,911."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used AGI of $98,411, which wrongly subtracts the spouse's $7,789 ESI premium input from wages. It also used a $29,250 MFJ standard deduction instead of NC's $25,500."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $6,787 at 3.99% implies about $170,000 of NC taxable income, far above the household's $106,200 AGI, so it inflated the income base instead of computing $106,200 − $25,500."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $98,410.90 is exactly $7,789 below the correct $106,199.90, which means it treated the spouse's ESI premium input as a pretax wage reduction. That input does not reduce AGI, so taxable income should be $80,700, not $72,911."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"$2,919.20 corresponds to an AGI of $98,663. That figure comes from subtracting the spouse's $7,789 ESI premium, which does not reduce AGI, and disallowing the head's $252 spousal IRA, which is deductible. The correct base is $80,700 of taxable income."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,514 at 3.99% implies about $88,000 of taxable income, roughly $7,400 above the correct $80,700. The answer is consistent with not taking the spouse's 401(k) or other adjustments out of AGI."
+us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It counted only $28,014 of SE income (the 92.35% SE-tax base) in AGI instead of the full $30,335. That understated AGI by $2,320.53 ($103,879 vs $106,200). The 0.9235 factor applies only to computing SE tax, not to the income included in AGI."
+us,scenario_089,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($63,098), which already reflects the federal standard deduction and QBI deduction. NC starts from federal AGI ($106,200). It also used an outdated $21,500 MFJ standard deduction instead of $25,500."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It returned zero tax even though AGI is $106,200 and only $25,500 is deductible for NC. The resulting $80,700 of taxable income is taxed at 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"$3,055 at 3.99% implies about $76,600 of taxable income, or an AGI near $102,000. That is ~$4,100 below the correct $106,200, so it took retirement adjustments beyond the spouse's 401(k) and the $577 of IRA deductions."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It explicitly subtracted 'pretax ESI premiums' from the spouse's wages, removing $7,789 and producing AGI of $98,411. The ESI premium input does not reduce AGI, so taxable income should be $80,700, not $72,911."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its NC taxable income of $69,696 implies an AGI of about $95,200, roughly $11,000 below the correct $106,200. The gap is consistent with subtracting the ESI premium input and other non-deductible amounts from wages."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$3,230 equals 3.99% × $80,952, which is $252 too high. It disallowed the head's traditional IRA contribution, which is deductible as a spousal IRA based on the spouse's compensation."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its AGI of $98,410.90 is exactly $7,789 below the correct $106,199.90, because it removed the spouse's ESI premium input from wages as a pretax deduction. With the correct AGI, NC taxable income is $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $98,410.90, which wrongly subtracts the spouse's $7,789 ESI premium input from wages. The correct AGI is $106,200, giving $80,700 of NC taxable income."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,120 at 3.99% implies about $78,200 of taxable income, or an AGI near $103,700. That is ~$2,500 below the correct $106,200 after the spouse's 401(k), half of SE tax and $577 of IRA deductions."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It had the correct AGI ($106,200) and standard deduction ($25,500) but applied 4.25%, NC's 2025 rate. The 2026 rate is 3.99%, giving $3,219.94 instead of $3,430."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly reached $80,700 of taxable income but applied the 2025 rate of 4.25%. The 2026 rate is 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied a 4% rate instead of 3.99%. It also understated AGI as $103,664 instead of $106,200, so taxable income came out at $78,164 instead of $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It started from the correct ~$106,200 AGI but used an MFJ standard deduction of about $26,700 instead of NC's $25,500. That reduced taxable income to ~$79,500 instead of $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no parseable value or explanation for state_income_tax_before_refundable_credits.
+us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $102,732.90 is exactly $3,467 below the correct $106,199.90, so it took the FLSA overtime premium out of AGI. The federal qualified-overtime deduction is below the line and does not reduce AGI, and NC's computation starts from AGI with no overtime subtraction."
+us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied NC's 4.5% rate from 2024 instead of the 2026 rate of 3.99%. It also understated AGI as $99,409 instead of $106,200."
+us,scenario_089,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI of $100,797 subtracts the head's $5,403 traditional 401(k), but the head has no wages, so no elective deferral can reduce income. Only the spouse's $6,946 deferral counts, which gives AGI of $106,200 and taxable income of $80,700."
+us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $3,467 FLSA overtime premium on top of gross wages that already include overtime. It also subtracted the head's $5,403 401(k) although the head has no wages, and it applied a 4.5% rate instead of the 2026 rate of 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It invented a $248,485 AGI, more than double the household's $106,200, along with a $32,200 standard deduction and a 4.5% rate. It also invented a credit for taxes paid to other states, though nothing in the facts sources any income outside North Carolina."
us,scenario_089,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_090,federal_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model treated the 22-year-old as eligible for the childless EITC, missing its minimum age of 25. Its subsequent phase-in and phaseout estimate is irrelevant because age disqualifies the filer before the credit schedule is applied."
us,scenario_090,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model applied the childless EITC schedule to a 22-year-old even though the minimum qualifying age is 25. It also submitted $721 after calculating a $535.50 phase-in amount, but the controlling error is that no EITC schedule applies to this filer."
@@ -7359,49 +7547,50 @@ us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,age_disability,F
us,scenario_090,state_refundable_credits,grok-4.5,llm_error,age_disability,False,"It computed the Kansas refundable EITC as 17% of a $536 federal EITC without applying the age-25 minimum for the childless federal EITC; at age 22 the head's federal EITC is $0, making the Kansas EITC $0. Its Kansas rate and arithmetic were right, but the federal base it multiplied was disallowed by the age test."
us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied only the Kansas food sales tax credit's income ceiling and skipped the categorical qualification, which limits the credit to filers who are 55 or older, blind or disabled, or have a dependent child under 18 living with them; a 22-year-old single filer with no dependents satisfies none of those, so the credit is $0. It further treated the credit as refundable, but Kansas made it nonrefundable beginning in tax year 2013, so even a qualifying $125 credit would not appear in state_refundable_credits."
us,scenario_090,tanf,glm-5.3,llm_error,categorical_eligibility,False,"Kansas TANF requires a dependent child in the assistance unit, and this 22-year-old head is a childless adult, so the benefit is $0 regardless of income; the household also holds $20,300 in bank assets, far above Kansas's TANF resource limit. glm-5.3 ran no eligibility test at all - it copied the $4,800 'financial assistance' input, a separate unearned-income variable in PolicyEngine rather than a TANF payment, straight into the tanf output. Its own stated reasoning ('financial assistance during the year, which is its annual TANF cash assistance amount') shows the substitution, and the prompt's instruction not to infer unlisted benefit receipt forbids reclassifying that input as TANF."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified overtime deduction, so ordinary taxable income came out at $14,945 instead of $14,317. That gives $1,545 − $120 = $1,425 rather than $1,470 − $120 = $1,350. It then submitted $1,276, which does not follow from its own $1,425 working."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified overtime deduction and taxed the $1,170 of capital gains at ordinary rates instead of in the 0% band. That put ordinary income at $16,115 instead of $14,317, adding about $216 of 12%-bracket tax ($1,685.80 − $119.90 instead of $1,470 − $120)."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It claimed a $2,500 American Opportunity Credit even though qualified tuition expenses are $0, which makes the AOTC $0. It also used the old $14,600 standard deduction, did not exclude the $926 traditional 401(k), taxed qualified dividends at 15% instead of 0%, and left out the overtime deduction and the $120 saver's credit. Its $650 does not match its own $490 working."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It taxed the $1,170 of capital gains as ordinary income, left out the $628 overtime deduction, and said no nonrefundable credits apply, so it missed the $120 saver's credit on $1,199 of retirement contributions. Its $1,656 does not even match its own $1,686 tentative tax."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed the $27,000 wages were already net of the $926 traditional 401(k), which overstated AGI at $37,109. It also taxed the $1,170 of gains as ordinary income and left out the $628 overtime deduction. It then subtracted an arbitrary $331 AOTC even though qualified expenses are $0, and it never applied the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $628 overtime deduction (ordinary income $14,945 instead of $14,317) and the $120 saver's credit. It then abandoned its own ~$1,684 bracket result and submitted an unsupported $2,109."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used pre-OBBBA 2026 parameters: a $15,000 standard deduction instead of $16,100 and a $11,925 top for the 10% bracket instead of $12,400. It also left out the $628 qualified overtime deduction, and it never considered the $120 saver's credit on $1,199 of 401(k)/IRA contributions."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It explicitly said the saver's credit does not reduce the tax, and it used a $1,500 nonrefundable AOTC even though qualified education expenses are $0. It then drifted to $1,470, which is the pre-credit regular tax, without subtracting the $120 saver's credit (10% of $1,199) that brings it to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction for the $628 FLSA overtime premium. Ordinary taxable income stayed at $14,945 instead of $14,317, which overstates tax by $75.50 ($1,545.40 instead of $1,470 before the $120 saver's credit)."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 OBBBA $16,100 and left out the $628 qualified overtime deduction. That inflated ordinary taxable income to $15,645 instead of $14,317, so regular tax came out at $1,634.90 instead of $1,470 before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It subtracted the $1,800 of other health insurance premiums from wages, used a $15,000 standard deduction instead of $16,100, left out the $628 overtime deduction, and used $11,925 as the top of the 10% bracket instead of $12,400. These errors roughly cancel ($14,245 of ordinary income vs. $14,316.66). Its unrounded $1,470.90 − $119.90 gives $1,351 instead of $1,470 − $120 = $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 $16,100, a wrong 10% bracket width, and left out the $628 qualified overtime deduction. That produced $15,645 of ordinary taxable income and $1,632.90 of tax instead of $14,317 and $1,470, before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 $16,100 and left out the $628 qualified overtime deduction. That raised taxable income to $20,883 and regular tax to $1,645.40 instead of $1,470 before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It wiped out the tax with a nonrefundable AOTC even though qualified education expenses are $0, which makes the AOTC $0. It also treated the $6,589 ESI premiums as a pre-tax wage reduction and applied a TCJA-sunset standard deduction plus personal exemption instead of the $16,100 OBBBA standard deduction. As a result, it never reached the $1,470 tax less the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $25,500 is about $7,200 too high: the correct figure is $18,285 after the $16,100 standard deduction and $628 overtime deduction, with only $14,317 taxed at ordinary rates. It also never subtracted the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It cut wages to $20,411 by treating the $6,589 employer-sponsored insurance premiums as pre-tax, but only the $926 traditional 401(k) reduces the $27,000 wages. It also assumed the TCJA sunset (pre-TCJA standard deduction plus personal exemption) instead of the permanent $16,100 OBBBA standard deduction, and it left out the $628 overtime deduction. The $120 saver's credit was the only step it got right."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It treated the $6,589 ESI premiums as a pre-tax exclusion (AGI $29,594 instead of $35,013) and applied an $8,300 standard deduction plus $5,300 exemption for a TCJA sunset that OBBBA prevented. It left out the $628 overtime deduction and never subtracted the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no computation. Its $1,475 roughly matches the $1,470 pre-credit tax on $14,317 of ordinary income, with no subtraction for the $120 saver's credit (10% of $1,199 in 401(k)/IRA contributions) that brings the answer to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted the $6,589 of employer-sponsored insurance premiums from wages and applied a personal exemption under an assumed TCJA sunset instead of the $16,100 OBBBA standard deduction plus the $628 overtime deduction. It also never applied the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation computes $1,756 on AGI of $36,183 using only the standard deduction, leaving out the $628 qualified overtime deduction and the $120 saver's credit. It then submitted $1,086, which matches neither its own explanation nor the correct $1,470 − $120."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and a $11,925 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 overtime deduction. That gave $1,686.90 before the saver's credit. It then submitted $1,617, which contradicts its own $1,567 result."
-us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,200 top for the 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 qualified overtime deduction. That produced $1,633.40 of regular tax instead of $1,470 before the correctly computed $119.90 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It said no nonrefundable credits apply after checking only the AOTC and EITC, which misses the $120 saver's credit on $1,199 of 401(k)/IRA contributions. It also used a $15,200 standard deduction instead of $16,100 and left out the $628 overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the standard deduction and AOTC eliminate the tax. But the $16,100 standard deduction leaves $18,285 of taxable income ($1,470 of tax), and the AOTC is $0 because no qualified education expenses are listed. Only the $120 saver's credit applies, giving $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It used an American Opportunity Credit to zero out the tax, but qualified education expenses are $0, so the AOTC is $0. The $14,317 of ordinary taxable income produces $1,470 of tax, reduced only by the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It got the $1,470 regular tax exactly right but said there were no usable nonrefundable credits. That misses the saver's credit: 10% of the $1,199 in traditional and Roth 401(k)/IRA contributions at AGI of about $35,013 is $120, which lowers the tax to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It offset the whole tax with a nonrefundable AOTC even though qualified tuition expenses are $0, which makes the AOTC $0. The correct result is $1,470 of regular tax less only the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the standard and overtime deductions but put only the qualified dividends in the 0% band. Taxing the $1,170 of capital gains at 12% added about $140. It also left out the $120 saver's credit, so it got $1,610 instead of $1,470 − $120 = $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the $6,589 employer-sponsored insurance premiums as a pre-tax wage reduction, cutting AGI to $29,594 and ordinary taxable income to $7,728. Wages are reduced only by the $926 traditional 401(k), which gives $14,317 of ordinary income and $1,470 of tax before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It got the $1,470 regular tax right but computed the saver's credit on only the $969 of traditional contributions ($96.90). It left out the $163 Roth 401(k) and $67 Roth IRA contributions, which also count; the full $1,199 base gives a $120 credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It computed the $1,470 regular tax correctly but applied no nonrefundable credits. It missed the saver's credit of 10% × $1,199 in 401(k)/IRA contributions = $120, which lowers the tax to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages as pre-tax, which cut ordinary taxable income to $7,728 instead of $14,317. It also said no credits were usable, which misses the $120 saver's credit on $1,199 of retirement contributions."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It claimed the AOTC and other nonrefundable credits wipe out the liability, but the AOTC is $0 because qualified education expenses are $0. The $18,285 of taxable income carries $1,470 of tax, and only the $120 saver's credit reduces it."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a personal exemption, a smaller standard deduction and a 15% bracket. OBBBA made the TCJA structure permanent, so the correct rules are a $16,100 standard deduction, no exemption and 10%/12% brackets, plus the $628 overtime deduction. Only its $120 saver's credit is correct."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (an $8,300 standard deduction, a $5,300 exemption and a 15% bracket) instead of the OBBBA-permanent $16,100 standard deduction with 10%/12% brackets, and it left out the $628 overtime deduction. That produced $2,017 of regular tax instead of $1,470 before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct $14,317 of ordinary taxable income and the saver's credit. However, it used $12,200 as the top of the 10% bracket instead of the 2026 $12,400, which overstates regular tax by $4 ($1,474.04 instead of $1,470)."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted the $6,589 ESI premiums from wages and applied a TCJA-sunset $8,350 standard deduction plus $5,330 exemption instead of the $16,100 OBBBA standard deduction plus the $628 overtime deduction. It also said no nonrefundable credits apply, which misses the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the pre-TCJA $8,300 standard deduction, a $5,200 exemption and a 15% bracket instead of the OBBBA-permanent $16,100 standard deduction and 10%/12% brackets, and it left out the $628 overtime deduction. It also cut the saver's credit to $97 by leaving out the Roth 401(k) and Roth IRA contributions from the $1,199 base."
-us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $1,350 derived from $1,470 of regular tax less the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used the American Opportunity Credit to fully offset the liability, but qualified education expenses are $0, so the AOTC is $0. The $1,470 regular tax is reduced only by the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% saver's credit rate. At AGI of about $35,013, a single filer is in the 10% tier, which gives $120 on the full $1,199 of traditional plus Roth contributions. Instead it took 50% of only $969 of traditional contributions ($484.50). It also left out the $628 overtime deduction and taxed the $1,170 of gains as ordinary income."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It invented a $1,960 AOTC to zero out the tax even though qualified education expenses are $0, which makes the AOTC $0. It also added the $628 overtime premium on top of the $27,000 wages, which already include it. The correct result is $1,470 of tax less the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the single standard deduction reduces taxable income to $0, which contradicts its own $40,571 figure. The $16,100 standard deduction and $628 overtime deduction leave $18,285 of taxable income, producing $1,470 of tax less the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It never applied the $628 qualified-overtime deduction. Its own math reached $1,545 − $120 = $1,425, about $75 too high because of that missing deduction, and it then submitted $1,276, which none of its arithmetic supports."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It taxed the $1,170 of non-Schedule-D capital gains as 12% ordinary income, which the reference keeps out of taxable income, and it skipped the $628 qualified-overtime deduction. Together that is $215.90 too much (12% × $1,798), giving $1,565.90 instead of $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It used a stale $14,600 standard deduction, taxed qualified dividends at 15% when they fall in the 0% band, and claimed a $2,500 AOTC even though no qualified tuition is listed, so the AOTC is $0. The submitted $650 doesn't follow from any of its steps. It also left out the overtime deduction and the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It wiped out all liability with an AOTC, but the AOTC is $0 with no qualified tuition expenses. Taxable income is $18,284.66 after the $16,100 standard deduction and $628 overtime deduction, which produces $1,470 of tax, and only the $120 Saver's Credit offsets it."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It taxed the $1,170 non-Schedule-D gains as ordinary income, skipped the $628 overtime deduction, and said no nonrefundable credits applied, missing the $120 Saver's Credit on $1,199 of contributions. It then submitted $1,656, which doesn't match its own $1,686."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It treated $27,000 as already net of the $926 traditional 401(k), so it never excluded the deferral. It also omitted the $628 overtime deduction and subtracted an arbitrary $331 of AOTC, even though no qualified tuition is listed and the AOTC is $0. It never applied the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It computed about $1,684 before credits without the $628 overtime deduction, then raised the answer to $2,109 with no stated basis. It also never applied the $120 Saver's Credit. The correct figure is $1,470 of tax minus $120."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and an $11,925 top for the 10% bracket instead of the 2026 figures of $16,100 and $12,400. It also omitted the $628 qualified-overtime deduction and the $120 Saver's Credit, ending at $1,687."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It used a $15,750 standard deduction with a $12,000 10% bracket and no overtime deduction. It then subtracted a $1,500 nonrefundable AOTC with no qualified tuition listed and arbitrarily 'adjusted' to $1,470. It also said the Saver's Credit didn't apply, when the $120 credit is what takes $1,470 down to $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It never applied the 2026 qualified-overtime deduction for the $628 FLSA overtime premium. That left ordinary taxable income $628 too high and tax about $75 too high ($1,425.50 instead of $1,350)."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 $16,100 and omitted the $628 qualified-overtime deduction, which overstated ordinary taxable income by about $1,328. Its Saver's Credit of $119.90 was right, but the base tax came out to $1,635 instead of $1,470."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $1,800 of non-employer health premiums from wages as if they were pre-tax. It also used the outdated $15,000 standard deduction and $11,925 bracket and left out the $628 overtime deduction. Those errors happened to net out close, but its unrounded $1,470.90 tax minus $119.90 gives $1,351, not the reference's $1,470 − $120 = $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,225 10% bracket instead of the 2026 $16,100 and $12,400, and it omitted the $628 qualified-overtime deduction. Ordinary taxable income came out at $15,645 instead of $14,316.66."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 $16,100 and never applied the $628 qualified-overtime deduction. That inflated the base tax to $1,645 instead of $1,470 before the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It excluded the $6,589 of ESI premiums from wages, used a TCJA-sunset standard deduction plus personal exemption, and offset all tax with an AOTC that is $0 when no qualified tuition is listed. The only nonrefundable credit available is the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $25,500 is far above the correct $18,284.66, which reflects the $16,100 standard deduction, the $628 overtime deduction, and $3,968 of 0%-rate qualified dividends. Its $1,399 doesn't come from the $1,470 bracket tax minus the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It cut wages to $20,411 by excluding the $6,589 of ESI premiums, and it applied a pre-TCJA standard deduction and personal exemption even though the 2026 single standard deduction is $16,100 with no exemption. It also omitted the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset ($8,300 standard deduction plus $5,300 exemption) instead of the 2026 $16,100 standard deduction, and excluded the $6,589 of ESI premiums from AGI. It also omitted the $628 overtime deduction and the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no derivation. The $1,475 sits at the $1,470 tax-before-credits level, so it is consistent with leaving out the $120 Saver's Credit (10% of $1,199 in retirement contributions)."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It excluded the $6,589 of ESI premiums from wages and applied a standard deduction plus personal exemption under the TCJA sunset, instead of the 2026 $16,100 standard deduction and $628 overtime deduction. It also applied no Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation ends at 1756 but it submitted 1086, so the two contradict each other. Neither figure applies the $628 qualified-overtime deduction or the $120 Saver's Credit that lead to $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the outdated $15,000 standard deduction and $11,925 10% bracket instead of the 2026 $16,100 and $12,400, and omitted the $628 overtime deduction. Its explanation reaches $1,567, yet it submitted $1,617."
+us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,200 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 qualified-overtime deduction. That made ordinary taxable income $15,645 instead of $14,316.66."
+us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It said no nonrefundable credits applied and so missed the $120 Saver's Credit on $1,199 of retirement contributions. It also used a $15,200 standard deduction instead of $16,100 and omitted the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the standard deduction and an AOTC eliminated the tax. In fact, $35,012.56 of AGI leaves $18,284.66 taxable after the $16,100 standard deduction and $628 overtime deduction, and the AOTC is $0 with no qualified tuition listed."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It used an AOTC to zero out the tax, but the AOTC is $0 because no qualified tuition is listed. That leaves $1,470 of bracket tax on $14,316.66 of ordinary income, offset only by the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It computed the $1,470 tax correctly with the $16,100 standard deduction and $628 overtime deduction, but said there were no usable nonrefundable credits. It missed the $120 Saver's Credit (10% of $1,199 in traditional and Roth contributions at AGI around $35k)."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It offset all tax with a nonrefundable AOTC that is $0 because no qualified tuition is listed. The only nonrefundable credit is the $120 Saver's Credit against $1,470 of tax."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It gave only qualified dividends the 0% rate and taxed the $1,170 non-Schedule-D gains as 12% ordinary income, adding about $140. It also omitted the $120 Saver's Credit, which yields $1,610 instead of $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the $6,589 of ESI premiums as a pre-tax exclusion from wages and so cut AGI to $29,594. Those premiums are not excluded, AGI is $35,012.56, and ordinary taxable income is $14,316.66 rather than $7,728."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It figured the Saver's Credit on only the $969 of traditional 401(k) and IRA contributions, which gives $96.90. It left out the $163 Roth 401(k) and $67 Roth IRA contributions, which also count, so the credit is $120 on $1,199."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It got the $1,470 tax right but said no other usable credits existed. It missed the $120 nonrefundable Saver's Credit on $1,199 of retirement contributions at AGI in the 10% tier."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It excluded the $6,589 of ESI premiums from wages, which understated AGI by $6,589, and it also omitted the $120 Saver's Credit. The result was $772.80 instead of $1,470 − $120."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It claimed the standard deduction plus an AOTC eliminated liability, but the AOTC is $0 with no qualified tuition listed. $18,284.66 of taxable income remains and produces $1,470 of tax before the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a personal exemption, the restored 15% bracket, and a smaller standard deduction. 2026 law instead has a $16,100 standard deduction, no exemption, a 12% second bracket, and the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-sunset law (about $8,300 standard deduction, $5,300 exemption, 15% bracket) instead of the 2026 $16,100 standard deduction with 10%/12% brackets. It also omitted the $628 qualified-overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It got the $14,317 of ordinary taxable income and the $119.90 Saver's Credit right, but capped the 10% bracket at $12,200 instead of the 2026 $12,400. That shifted $200 into the 12% bracket and gave $1,474 instead of $1,470."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It excluded the $6,589 of ESI premiums from wages and applied a TCJA-sunset $8,350 standard deduction plus a $5,330 exemption instead of the 2026 $16,100 standard deduction and $628 overtime deduction. It also skipped the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (about $8,300 standard deduction, $5,200 exemption, 15% bracket) and omitted the $628 overtime deduction. It also computed the Saver's Credit as $97 on traditional contributions only, when the correct credit is $120 on $1,199 that includes the Roth contributions."
+us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used an AOTC to zero out the tax, but no qualified tuition is listed, so the AOTC is $0. That leaves $1,470 of tax offset only by the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied the 50% Saver's Credit rate, but AGI of about $35k is above the 2026 single 50%-tier limit ($24,250) and falls in the 10% tier. It also left the Roth contributions out of the base, taxed the $1,170 gains as ordinary income, and omitted the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It added the $628 overtime premium on top of $27,000 in wages, which already include it, instead of deducting it as qualified overtime. It then zeroed the tax with an AOTC that is $0 because no qualified tuition is listed."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed a single filer's standard deduction cut $40,571 of income to $0 taxable income, which is wrong because the 2026 standard deduction is only $16,100. Correct taxable income is $18,284.66, producing $1,470 of tax before the $120 Saver's Credit."
us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached the correct result — no qualified tuition expenses are listed, so under the prompt's unlisted-inputs-are-zero rule the American Opportunity Credit and its 40% refundable portion are $0 — and it then overrode that derivation to submit $400 as a 'minimal refundable AOTC assumption'. It also mis-stated the EITC bar as the AGI phase-out ceiling while never applying the age-25 floor that independently zeroes the childless EITC for this 22-year-old."
us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated the AOTC administrative flags (1098-T, institution EIN, half-time enrollment, credential pursuit) as if they established up to $4,000 of qualified expenses, when tuition is an unlisted input equal to $0, so the credit computed as 100% of the first $2,000 plus 25% of the next $2,000 is $0 and the refundable portion is $0. It then submitted $1,200, exceeding the $1,000 statutory refundable cap it had itself just cited."
us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It jumped from AOTC eligibility flags straight to the $2,500 statutory maximum and took 40% of it, never computing the credit base: qualified tuition expenses are $0 here, so the credit and its refundable portion are $0. Its EITC rationale is also wrong on its own numbers — $10,152 of investment income is below the disqualifying limit — and it missed the actual disqualifier, the age-25 floor for childless EITC claimants."
@@ -7431,52 +7620,53 @@ us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was nothing to score. The correct answer is 7.65% of $27,000 in wages, or $2,065.50."
us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of the $27,000 gross wages, taxing $27,628. The prompt states that gross wages already include overtime pay, so this counts the overtime twice. The correct base is $27,000, giving $2,065.50."
us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It put the $1,170 capital gain distributions into WI income (net of a 30% exclusion). It estimated the sliding standard deduction at ~$10,668 from an old $12,760 base, versus the reference's $12,067.66, and never subtracted the $700 personal exemption. Its own arithmetic came to ~$975, but it submitted $1,257, which nothing in its derivation supports."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,170 capital gain distributions in WI AGI ($36,183 vs $35,012.56) and left out the $700 personal exemption, which gave taxable income of ~$24,184 instead of $22,244.90. It also used the 2025 first-bracket width of $14,680 rather than the 2026 figure of ≈$15,010."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It left all $8,982 of interest and dividends out of income and used the federal $14,600 standard deduction instead of WI's sliding-scale deduction and $700 exemption. It then zeroed the tax by applying the federal American Opportunity Credit to Wisconsin liability. Wisconsin has no AOTC counterpart and no qualified tuition is listed, so the $843.66 on $22,244.90 of taxable income has no nonrefundable offset."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It guessed the phased standard deduction at ~$9,500, far below the $12,067.66 the sliding scale gives at $35,012.56 of WI income. It also included the $1,170 capital gain distributions, so taxable income came to ~$25,983 instead of $22,244.90. It then raised its own computed $1,014 to $1,140 without giving a reason."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It estimated the phased standard deduction at only ~$7,700 (reference $12,067.66) and included the $1,170 capital gain distributions, giving taxable income of ~$27,783 versus $22,244.90. It then added an unexplained ~$70 to its computed $1,093 to reach $1,163."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put the phased-down standard deduction at ~$9,000 instead of $12,067.66 and started from $36,183, which includes the $1,170 capital gain distributions. That made taxable income ~$26,500 rather than $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of the $1,170 non-Schedule-D capital gain distributions in WI income after a 30% exclusion, but the reference leaves those distributions out of gross income entirely. That raised WI income to $35,832 and taxable income to ~$23,129 versus $22,244.90, putting it ~$38 too high."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It carried 2024-style parameters into 2026: a $12,230 standard-deduction base phased down from $15,000, which left only $9,688 versus the reference's $12,067.66, and a $13,810 first bracket instead of ≈$15,010. With the $1,170 capital gain distributions also included, taxable income came to $25,795 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions, underestimated the sliding standard deduction at ~$11,340 (reference $12,067.66), and used a $14,320 first bracket, reaching $933. It then subtracted a made-up 'Married/Single credit' and 'personal exemption credit' to get $900. Wisconsin's $700 exemption is a deduction it had already taken, and no nonrefundable credit applies."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its ~$23.9k of taxable income ($36.2k minus $11.6k minus $0.7k) counts the $1,170 capital gain distributions as income and uses a standard deduction ~$470 below the reference's $12,067.66. Together these overstated taxable income by ~$1,650 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $13,450 standard deduction without the 12% sliding-scale phase-down that cuts it to $12,067.66 at this income, and it included the $1,170 capital gain distributions in AGI. It also used a $14,320 first bracket instead of ≈$15,010. These errors partly cancel, leaving $840.57 instead of $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-2023 rates of 3.54% and 4.65% instead of Wisconsin's 3.5% and 4.4%. It also took an unphased $13,230 standard deduction, skipped the $700 exemption, and started from an unexplained $34,383 AGI instead of $35,012.56."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used an unphased $14,050 standard deduction and left out the $700 personal exemption; the reference applies the sliding-scale $12,067.66 plus $700. It also included the $1,170 capital gain distributions and set the first bracket at $15,200. These errors partly cancel, leaving $837.05."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions, phased the standard deduction down to only $9,963.40 (reference $12,067.66), and never subtracted the $700 personal exemption. That gave taxable income of $26,219.60 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages as if they were pre-tax, cutting AGI to $29,594. The reference reduces wages only by the $926.16 401(k) deferral, for WI income of $35,012.56. The understated income left taxable income at $17,484 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It based WI tax on $27,000 of wages alone. That skips the 401(k) exclusion, the $4,806 of interest, and the $4,176 of dividends that make WI income $35,012.56. It showed no standard deduction, exemption, or bracket calculation behind $905."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It showed no figures. $588 corresponds to about $16,300–16,450 of taxable income, which matches the pattern of subtracting the $6,589 employer insurance premiums from wages (AGI ≈ $29,594). The reference instead uses $35,012.56 of WI income and $22,244.90 of taxable income."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $6,589 employer-sponsored insurance premiums from income, giving AGI of $29,594 instead of $35,012.56, and left out the $700 personal exemption. Taxable income came to $16,887 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $1,177 implies roughly $29,700 of taxable income, which means it taxed ~$36k of income with little of the $12,067.66 sliding standard deduction and none of the $700 exemption."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 60% capital gain exclusion; Wisconsin's is 30%, and the reference keeps the $1,170 non-Schedule-D distributions out of gross income altogether. It also subtracted a medical expense credit that Wisconsin does not give on these facts, pulling the tax down to $543.09 instead of $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and its explanation says 1,018 while it submitted 944. Both figures are above $843.66, which fits counting the $1,170 capital gain distributions and understating the $12,067.66 sliding standard deduction."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $1,113 implies about $28,300 of taxable income, so it applied far less than the $12,067.66 sliding standard deduction and $700 exemption against WI income of $35,012.56."
-us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It treated the WI standard deduction as fully phased out at $36,183, but the sliding scale still allows $12,067.66 at $35,012.56 of income, and it also skipped the $700 exemption. It then taxed the whole AGI, including the $1,170 capital gain distributions, reaching a 5.3% bracket that $22,244.90 of taxable income never touches."
-us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the full $36,183 as though the standard deduction were fully phased out, and left out the $700 exemption. At $35,012.56 of WI income the sliding scale still gives $12,067.66, leaving $22,244.90 of taxable income."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the tax was eliminated without computing it. WI income of $35,012.56 minus the $12,067.66 standard deduction and $700 exemption leaves $22,244.90 taxable, which produces $843.66, and no nonrefundable credit offsets it."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no state AOTC and no qualified tuition is listed, so nothing offsets the $843.66 computed on $22,244.90 of taxable income."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 FLSA overtime premium and the $1,800 of other health insurance premiums from WI income, and applied a 30% exclusion to the capital gain distributions, arriving at $33,404. The reference makes neither the overtime nor the premium subtraction and leaves the non-Schedule-D distributions out entirely, giving $35,012.56. The understated income pulled taxable income down to ~$20,190 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $943 implies ~$24,500 of taxable income, about $2,250 above the reference's $22,244.90, which fits counting the $1,170 capital gain distributions and underestimating the $12,067.66 sliding standard deduction."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It kept the $1,170 non-Schedule-D capital gain distributions in WI income net of a 30% exclusion, while the reference leaves them out of gross income. Combined with its approximate 2026 deduction and bracket figures, this landed $3.34 above $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $860 implies ~$22,600 of taxable income, about $370 above the reference's $22,244.90, which fits including part of the $1,170 capital gain distributions that the reference leaves out of WI income."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages plus another $1,800 health-insurance deduction, cutting WI income to $27,443 versus $35,012.56. Taxable income fell to $13,714.96, all of it taxed at 3.5%."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 capital gain distributions the reference leaves out. Its deduction estimate left roughly $24,300 of taxable income against $22,244.90, overstating the tax by ~$92."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $36,183 of income, including the $1,170 capital gain distributions the reference leaves out, and its sliding standard deduction left roughly $23,900 taxable versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages plus another $1,800 insurance deduction, cutting WI income to $27,443 versus $35,012.56. That left only $13,769.76 of taxable income, all taxed at 3.5%, instead of $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the standard deduction and credits fully offset the tax. The $12,067.66 sliding standard deduction and $700 exemption still leave $22,244.90 taxable, and no WI nonrefundable credit applies because no rent, property tax, or other credit inputs are listed."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions (AGI $36,183) and estimated the standard deduction at ~$11,046 instead of $12,067.66, leaving ~$24,437 taxable versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of capital gain distributions after a 30% exclusion (WI income $35,832 vs $35,012.56) and underestimated the sliding standard deduction at ~$10,618 (reference $12,067.66). It also used the $14,320 first bracket instead of ≈$15,010, overstating taxable income by ~$2,270."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium to get federal AGI of $35,555, but the federal overtime deduction is taken below AGI and does not reduce AGI or WI income. It also kept $819 of the capital gain distributions after a 30% exclusion, giving WI income of $35,204 versus $35,012.56, and its parameter estimates account for the rest of the $15 overshoot."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It started from $29,594, which removes the $6,589 employer-sponsored insurance premiums from wages, and left out the $700 personal exemption. Taxable income came to $16,811 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used an old 4.65% rate for the second bracket; Wisconsin's second rate is 4.4%. It also included the $1,170 capital gain distributions and estimated the standard deduction at ~$10,900–11,000 versus $12,067.66, leaving taxable income at ~$24,500 versus $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of the $1,170 capital gain distributions after a 30% exclusion (WI income $35,832 vs $35,012.56), which also shrank its phased standard deduction to $11,868.47 versus $12,067.66. That produced taxable income of $23,263.53 versus $22,244.90, and it also put the first bracket at $15,163.62 rather than ≈$15,010."
-us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It offset the tax with the Wisconsin School Property Tax Credit, which requires property taxes or rent paid. None are listed, so the credit is zero and the $843.66 on $22,244.90 of taxable income stands."
-us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started the standard-deduction phase-down at ~$27,800, which is too high, giving ~$13,480 versus the reference's $12,067.66. It then left out the $700 exemption and included the $1,170 capital gain distributions. These errors partly cancel, leaving taxable income of ~$22,700 and tax of $863."
-us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted half of the $6,589 ESI premiums from wages as an 'employer portion' and used the pre-2023 rates of 3.54% and 4.65% instead of 3.5% and 4.4%. It also took an unphased standard deduction and turned the $700 personal exemption into a $77 credit. It then averaged arbitrary deduction guesses to reach $720.73."
-us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It overstated WI income at $40,571 (reference $35,012.56) and used an unphased $12,200 standard deduction. It then claimed unspecified nonrefundable credits wiped out its $1,374.94 of tax. No WI nonrefundable credit applies, so the tax on $22,244.90 of taxable income is $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,170 of capital gain distributions at 70%, giving $35,832 instead of Wisconsin AGI of $35,012.56. It phased the standard deduction down to about $10,668 instead of $12,067.66 and never subtracted the $700 personal exemption. It also submitted $1,257 even though its own math came to about $975; the correct taxable income of $22,244.90 gives $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of non-Schedule-D gains that are left out of the $35,012.56 Wisconsin AGI. It also skipped the $700 personal exemption, so taxable income came to $24,184 instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal taxable income using the federal standard deduction, rather than Wisconsin AGI with Wisconsin's sliding-scale deduction and $700 exemption. It also left out the $8,982 of interest and dividends. It then cut the tax to $0 with the federal American Opportunity Credit, which has no Wisconsin version; Wisconsin taxable income of $22,244.90 gives $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It said taxable income was low enough for $0 tax, ignoring $4,806 of interest and $4,176 of dividends. Those bring Wisconsin AGI to $35,012.56 and taxable income to $22,244.90 after the $12,067.66 standard deduction and $700 exemption, which is $843.66 of tax."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 of gains in a $36,183 AGI and phased the standard deduction down to about $9,500 instead of $12,067.66, which inflated taxable income to $25,983. It then submitted $1,140, above even its own $1,014."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It cut the sliding-scale standard deduction to about $7,700 instead of $12,067.66 and kept the $1,170 of gains in a $36,183 AGI, giving $27,783 of taxable income instead of $22,244.90. It then added about $70 for unnamed 'adjustments'."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It cut the standard deduction to about $9,000 instead of $12,067.66 and started from $36,183, which includes the $1,170 of gains. That gave about $26,500 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 in Wisconsin income ($35,832) after a 30% capital gain exclusion. The engine leaves these non-Schedule-D distributions out entirely, giving $35,012.56. The extra $819, plus a slightly low $12,003 deduction, raised taxable income to $23,129 instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used 2024 parameters: a $12,230 deduction phased out at 12% above $15,000. That cut the deduction to $9,688 instead of the 2026 sliding-scale amount of $12,067.66. Combined with the $1,170 of gains in a $36,183 AGI, taxable income came to $25,795 instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 of gains in a $36,183 AGI and used an $11,340 deduction instead of $12,067.66, giving $24,143 of taxable income instead of $22,244.90. It then subtracted a 'Married/Single credit' that does not apply to this single filer."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It phased the standard deduction down to about $11.6k instead of $12,067.66 and kept the $1,170 of gains in income. That gave about $23.9k of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used an unphased $13,450 standard deduction instead of the $12,067.66 phased amount, and kept the $1,170 of gains in a $36,183 AGI. These two errors roughly cancelled, giving $22,033 of taxable income instead of $22,244.90. It also used a $14,320 first bracket, which is not the 2026 threshold."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It used the old 3.54%/4.65% rates instead of 2026's 3.5%/4.4%. It applied an unphased $13,230 deduction, skipped the $700 exemption, and started from an unexplained $34,383 AGI instead of $35,012.56."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains that are left out of the $35,012.56 Wisconsin AGI. It subtracted a lump $14,050 instead of the $12,067.66 phased deduction plus $700 exemption, and used a $15,200 first bracket, giving $22,133 of taxable income and $837.05."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It cut the standard deduction to $9,963.40 instead of $12,067.66 and never subtracted the $700 personal exemption. Starting from a $36,183 AGI that includes the $1,170 of gains, it got $26,219.60 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $29,594 AGI equals $36,183 minus the $6,589 of employer-sponsored insurance premiums, which are not a wage reduction here. That drops Wisconsin income well below the correct $35,012.56, giving $17,484 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It treated Wisconsin AGI as $27,000 of wages alone. It left out the $926.16 401(k) deferral and the $4,806 of interest and $4,176 of dividends that bring Wisconsin AGI to $35,012.56. Its $905 does not follow from any stated deduction or bracket calculation."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"At 3.5%/4.4%, $588 implies about $16,400 of taxable income, roughly $5,800 below the correct $22,244.90. That matches subtracting the $6,589 of employer-sponsored insurance premiums from income, as other Gemini models did, instead of starting from the $35,012.56 Wisconsin AGI."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 of employer-sponsored insurance premiums to get a $29,594 AGI, instead of the correct $35,012.56. It also skipped the $700 personal exemption, giving $16,887 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"With no derivation given, $1,177 implies about $29,800 of taxable income, roughly $7,500 above the correct $22,244.90. That is consistent with phasing the standard deduction down far below the correct $12,067.66 from a $35,012.56 Wisconsin AGI."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 60% capital gain exclusion, which Wisconsin does not have; Wisconsin's exclusion is 30%, and the engine leaves these non-Schedule-D distributions out of income entirely. It also subtracted a 'medical expense credit' that Wisconsin does not allow on these facts, pushing the tax below the correct $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $944 while its explanation says $1,018. It treated the $700 exemption as a 'personal exemption credit'; Wisconsin allows it as a deduction from income. Both figures imply taxable income $2,300 to $4,000 above the correct $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"$1,113 implies about $28,400 of taxable income, roughly $6,100 above the correct $22,244.90. That is consistent with under-applying the sliding-scale standard deduction, which is still $12,067.66 at $35,012.56 of Wisconsin AGI."
+us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It said the Wisconsin standard deduction is fully phased out to $0 for a single filer at this income. At $35,012.56 of Wisconsin AGI it is still $12,067.66. It also skipped the $700 exemption, included the $1,170 of gains, and reached the 5.3% bracket on income that should be taxed only at 3.5%/4.4%."
+us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It said the standard deduction is fully phased out and taxed the whole $36,183. The sliding-scale deduction is still $12,067.66 at this income, and the $700 exemption also applies, giving $22,244.90 of taxable income."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It said Wisconsin tax was fully eliminated. Wisconsin AGI of $35,012.56, less the $12,067.66 standard deduction and $700 exemption, leaves $22,244.90 of taxable income, and no Wisconsin nonrefundable credit offsets the resulting $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no American Opportunity Credit, so the $843.66 on $22,244.90 of taxable income stands."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It cut Wisconsin income to $33,404 by subtracting the $628 FLSA overtime premium and the $1,800 of health premiums. Neither has a Wisconsin subtraction here. It also kept 70% of the capital gain distributions, while the correct Wisconsin AGI is $35,012.56 with a $12,067.66 deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"$943 implies about $24,500 of taxable income, roughly $2,260 above the correct $22,244.90. That is consistent with keeping the $1,170 of capital gain distributions in Wisconsin income and under-applying the $12,067.66 deduction or the $700 exemption."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied a capital gain subtraction to the $1,170 of distributions, which keeps part of them in Wisconsin income. The engine leaves them out entirely, giving $35,012.56 of Wisconsin AGI and $22,244.90 of taxable income. The extra income makes its $847 overshoot $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"$860 implies about $371 more taxable income than the correct $22,244.90. That matches leaving part of the $1,170 of capital gain distributions in Wisconsin income, when Wisconsin AGI should be $35,012.56."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Its $27,443 of Wisconsin income comes from subtracting the $6,589 of employer-sponsored insurance premiums and an $1,800 health insurance deduction. Neither belongs in Wisconsin AGI, which is $35,012.56. That drove taxable income down to $13,714.96 instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of non-Schedule-D gains left out of the $35,012.56 Wisconsin AGI. With an understated deduction, $936 implies about $24,300 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains left out of the $35,012.56 Wisconsin AGI. With a deduction below $12,067.66, $920 implies about $24,000 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $27,443 of Wisconsin income comes from subtracting the $6,589 of employer-sponsored insurance premiums and an $1,800 insurance deduction. Neither belongs in Wisconsin AGI, which is $35,012.56. That left $13,769.76 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It said the standard deduction and credits fully offset the tax. The $12,067.66 phased deduction and $700 exemption leave $22,244.90 of the $35,012.56 Wisconsin AGI taxable, and no Wisconsin nonrefundable credit applies."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains, and phased the deduction to about $11,046 instead of $12,067.66. That gave $24,437 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 in Wisconsin income ($35,832 instead of $35,012.56) and cut the deduction to $10,618 instead of $12,067.66. That gave $24,514 of taxable income instead of $22,244.90, taxed with an outdated $14,320 first bracket."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium from AGI and kept 70% of the $1,170 of gains, ending at $35,204 of Wisconsin income instead of $35,012.56. The roughly $350 of extra taxable income produced $859 instead of $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Its $29,594 Wisconsin AGI comes from subtracting the $6,589 of employer-sponsored insurance premiums, instead of using $35,012.56. It also skipped the $700 personal exemption, giving $16,811 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income above the first bracket at the repealed 4.65% rate instead of 4.4%. It also cut the standard deduction to about $11,000 instead of $12,067.66 and kept the $1,170 of gains in a $36,183 starting income."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no parseable value or explanation for state_income_tax_before_refundable_credits, so nothing could be compared with the $843.66 computed from $22,244.90 of Wisconsin taxable income."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 of capital gain distributions in Wisconsin income ($35,832 instead of $35,012.56) and phased the deduction to $11,868.47 instead of $12,067.66. That gave $23,263.53 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It wiped out the tax with the Wisconsin School Property Tax Credit. That credit is based on property tax or rent paid, and no rent or property tax is listed, so it is $0. The $843.66 on $22,244.90 of taxable income stands."
+us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains, used a $13,480 deduction instead of $12,067.66, and never subtracted the $700 exemption. That gave about $22,700 of taxable income instead of $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted half of the $6,589 employer insurance premiums from wages and used the old 3.54%/4.65% rates instead of 3.5%/4.4%. It treated the $700 exemption as a $77 credit, guessed at unphased deductions, and averaged the results, instead of using $35,012.56 Wisconsin AGI less the $12,067.66 deduction and $700 exemption."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It overstated Wisconsin income at $40,571 and then zeroed out its own $1,374.94 with unnamed nonrefundable credits. None apply to this filer, and the correct $22,244.90 of taxable income gives $843.66."
us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income."
us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction."
@@ -7519,116 +7709,119 @@ us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disab
us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model recited the correct statutory rule — 24 months of SSDI receipt confers Medicare — then discarded it, concluding that ""for tax-benefit modeling purposes, a disabled individual is treated as Medicare-eligible."" PolicyEngine's is_medicare_eligible implements no such shortcut: it is the age >= 65 test alone, which Dependent 1 at age 27 fails."
us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model stated a disjunctive engine rule — ""In PolicyEngine, Medicare eligibility is determined by age 65 or older OR disability status"" — and fired the second branch off the is_disabled flag. That OR branch does not exist in is_medicare_eligible, which evaluates age >= 65 only, so Dependent 1 at 27 is not eligible."
us,scenario_093,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly reached AGI of $112,408 and the $32,200 standard deduction, but then subtracted a $1,000 auto-loan-interest deduction that is not part of the computation. It then replaced its own $8,509 result with an unexplained $8,067 'parameter adjustment'. Without the auto-loan deduction, taxable income is $80,208.25, tax is $9,128.99, and the $500 ODC leaves $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It added the dependent's $45,000 wages to the joint return, which produced $124,208 of taxable income. The tax unit's gross income includes only the head's and spouse's wages net of 401(k) ($113,941.88) plus $1,156 interest, so AGI is $112,408.25 and taxable income is $80,208.25. It also took a $1,000 auto-loan-interest deduction that the computation does not include."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It pooled the dependent's $45,000 wages into the joint return, used the outdated $29,200 standard deduction and pre-2026 brackets, and omitted the $500 ODC. It then subtracted an arbitrary $4,246 for 'refundable credits and reconciliation', which does not belong in a before-refundable-credits measure. Using only the head's and spouse's income gives $80,208.25 of taxable income and $8,628.99 after the ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It rejected the 27-year-old as a dependent on support and gross-income grounds, but a permanently and totally disabled child is a qualifying child at any age with no gross income test, so the $500 credit for other dependents applies. It also used the 2025 $31,500 standard deduction and a $24,150 10% bracket instead of 2026's $32,200 and $24,800. With the correct figures, tax before the credit is $9,128.99, and $8,628.99 after it."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It applied the qualifying-relative gross income limit and concluded there is no dependent. The disabled 27-year-old is a qualifying child regardless of age or income, so the $500 ODC reduces tax. It also used a $31,500 standard deduction and a $24,000 10% bracket instead of 2026's $32,200 and $24,800, which overstated tax before credits ($9,229 versus $9,128.99)."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $32,600 standard deduction. After reaching $8,371 net of the ODC, it added an unexplained $2,000 for 'the disabled adult dependent's own tax'. The dependent's wages are not part of this tax unit's liability, so nothing is added; with the $2,500 cap and the $32,200 deduction, the result is $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It counted all three earners' wages ($163,000) in the joint return's income, which gave AGI of $157,408 and taxable income of $124,208. The joint return includes only the head's and spouse's income, which gives AGI of $112,408.25. It also took a $1,000 auto-loan-interest deduction that the computation does not allow; the correct taxable income is $80,208.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the qualifying-relative gross income test and declared no dependent, missing that a permanently disabled child is a qualifying child at any age, so the $500 ODC applies. It also used the 2025 pre-OBBBA $30,000 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800. With the correct figures, tax is $9,128.99 before the credit and $8,628.99 after it."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct $112,408 AGI but used an outdated ~$30,300 standard deduction instead of $32,200 and denied the $500 ODC for the disabled adult child. It then raised its own $9,376 computation to $10,800 with no stated basis. The correct taxable income is $80,208.25, which gives $9,128.99 of tax and $8,628.99 after the ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It ruled out the ODC because the 27-year-old earns $45,000, but a permanently disabled child is a qualifying child with no age or income limit, so the $500 credit applies. It also subtracted a $1,000 auto-loan-interest deduction that the computation does not include, which lowered taxable income to $79,208 instead of $80,208.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 wages in the joint return and then wrongly treated the IRA and student-loan deductions as phased out; the joint MAGI of about $115,000 is below both phase-out ranges. It also used a $30,600 standard deduction and omitted the $500 ODC for the disabled dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It subtracted the listed ESI premiums from wages, which pushed joint AGI below $112,408.25; the engine removes only the traditional 401(k) deferrals from wages. It then added a $2,354.82 single-return tax for the dependent, but the output is the one tax unit's liability, where the dependent is claimed (earning a $500 ODC) and the dependent's wages are not taxed."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It subtracted $8,389 of ESI premiums from each earner's wages and summed a married-filing-jointly return with a separate single return for the dependent, without the $500 ODC. The correct tax unit is the joint return alone: AGI $112,408.25 with no ESI reduction, $80,208.25 taxable, and $8,628.99 after the ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It cut joint AGI to $95,630 by subtracting the ESI premiums, which do not reduce wages in this computation, and then added $2,309.98 of tax from a separate single return for the dependent. The claimed dependent's wages are not part of the tax unit's liability; the joint return alone yields $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired after 2025 and applied personal exemptions and a $15,850 standard deduction for married filing jointly. OBBBA made the TCJA rate schedule permanent, with a $32,200 2026 joint standard deduction and no exemptions. It also subtracted ESI premiums from wages and added a separate tax for the dependent instead of claiming the $500 ODC on the single joint return."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It built taxable income from a $163,156 income base that includes the dependent's $45,000 wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, and omitted the $500 ODC. The joint return counts only the head's and spouse's income, which gives $80,208.25 of taxable income."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 parameters (personal exemptions and a lower standard deduction), although OBBBA made the TCJA brackets and the $32,200 joint standard deduction permanent. It also added a separate return's tax for the dependent instead of claiming that person as a dependent and taking the $500 ODC on the one joint return."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used 'under 2026 law' personal exemptions from an assumed TCJA sunset and subtracted ESI premiums to reach joint AGI of $95,630. It also added $3,006.65 of tax on a separate single return for the dependent. The correct computation has AGI of $112,408.25, the $32,200 standard deduction, TCJA brackets, and a $500 ODC, which gives $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $11,048 is about $2,419 above the correct $8,628.99 and closely matches pooling all three earners' wages net of ESI premiums into one joint return (about $100,041 taxable, $11,009 after the ODC). Only the head's and spouse's income belongs on the return, which gives $80,208.25 of taxable income."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It subtracted ESI premiums from wages in addition to 401(k) and IRA contributions, and it summed the tax from a separate return for the dependent with the joint return's tax. The claimed dependent's wages generate no liability in this tax unit, and ESI premiums do not reduce wages, so the joint return alone yields $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It computed tax for two filing units, a joint return plus a single return for the dependent, and summed them. The household is one tax unit in which the disabled 27-year-old is a claimed dependent whose wages are not taxed on the joint return and who generates a $500 ODC, giving $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It reached a household AGI of about $157,598 by including the dependent's $45,000 wages, and it subtracted personal exemptions that no longer exist under the permanent TCJA/OBBBA rules. The joint return's AGI is $112,408.25, and taxable income after the $32,200 standard deduction is $80,208.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It correctly claimed the $500 ODC on the joint return but then added $3,353.74 of tax from a separate return for the dependent; the claimed dependent's wages are not part of this tax unit's liability. It also used a $30,840 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800, which overstated the joint tax ($8,811 versus $8,628.99)."
-us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It treated the 27-year-old as a self-supporting separate filer, so it dropped the $500 ODC and added $3,210 of that person's own tax, but a permanently disabled child is a qualifying child regardless of income. It also deducted the full $3,000 of student loan interest instead of the $2,500 cap."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and retirement deductions eliminate all tax. However, AGI of $112,408.25 minus the $32,200 standard deduction leaves $80,208.25 of taxable income, which produces $9,128.99 of tax, or $8,628.99 after the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced income by 'itemized medical' and ESI-premium effects. Its $6,347 is about $2,282 below the correct amount, which matches removing roughly $19,000 (the $16,778 of ESI premiums plus the listed medical costs) at 12%. The ESI premiums do not reduce wages, and the approximately $2,200 of medical costs is far below the 7.5%-of-AGI floor under a standard-deduction return."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It put all three earners' wages on the joint return and subtracted all three ESI premiums, which produced $133,775 of wages and $132,241 of AGI. Only the head's and spouse's income counts, and ESI premiums do not reduce wages, so AGI is $112,408.25 and taxable income is $80,208.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 of taxable income includes the dependent's $45,000 wages, which are not part of the joint return. It also applied no nonrefundable credit, missing the $500 ODC for the disabled adult child. The correct result is $80,208.25 taxable, $9,128.99 of tax, and $8,628.99 after the credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It taxed household wages from all three earners on the joint return, pushing taxable income into the 22% bracket at about $125,000. The joint return includes only the head's and spouse's income, which gives $80,208.25 of taxable income entirely within the 10%/12% brackets."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its stated joint taxable income of $85,208 is exactly $5,000 above the correct $80,208.25 (AGI $112,408.25 minus the $32,200 standard deduction), which is equivalent to applying only a $27,200 deduction. The extra $5,000 taxed at 12% adds $600, turning the correct $8,628.99 into $9,228.96."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It combined all three workers' wages into one joint return and subtracted all three ESI premiums, reaching $132,241 of AGI. The dependent's wages are excluded from the tax unit's gross income and ESI premiums do not reduce wages, so AGI is $112,408.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It computed the joint tax correctly ($9,128.96) but treated the 27-year-old as a separate filer, adding $3,220 of their tax and dropping the $500 ODC. The disabled adult child is a claimed dependent in the single tax unit, so the answer is $9,128.99 minus $500."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,778 of listed employer-sponsored insurance premiums from the head's and spouse's wages, which lowered AGI to $95,630. The engine removes only the $4,058.12 of traditional 401(k) deferrals from wages, so AGI is $112,408.25. The extra $16,778 exclusion at 12% accounts exactly for the $2,013.39 shortfall."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It based the tax on 'household wages', pooling the dependent's $45,000 into the taxed income. The joint return taxes only the head's and spouse's wages net of 401(k) plus interest, which gives $80,208.25 of taxable income and $8,628.99 after the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules (personal exemptions and a 15% bracket), although OBBBA made the TCJA brackets and the $32,200 joint standard deduction permanent for 2026. It also added $4,846 of tax from a separate single return for the dependent instead of claiming the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a sunset-law $16,600 standard deduction, $5,300 exemptions, and a 15% bracket, but 2026 runs on the permanent TCJA schedule: a $32,200 standard deduction, 10% to $24,800, then 12%. It also added $4,130 from a separate single return for the dependent and omitted the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It added $3,224 of tax from a separate single return for the dependent and omitted the $500 ODC to which the claimed disabled adult child entitles the joint return. It also subtracted a $173 overtime-premium deduction, although neither earner works over 40 hours at a straight-time hourly rate."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'post-TCJA 2026 brackets' with a $30,750 standard deduction, which overstated the joint tax at $11,001. It then added $3,820 of tax from a separate single return for the dependent. Under the permanent TCJA schedule, taxable income of $80,208.25 yields $9,128.99, less the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($80,208 taxable, $9,129 of tax) but also added $3,220 from a separate single return for the 27-year-old and applied no ODC. The disabled adult child is a dependent in the same tax unit, so the answer is $9,128.99 minus the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It treated all three members as one joint filer with $163,000 of wages, including the dependent's $45,000, and it treated the traditional IRA deductions as fully phased out. Only the head's and spouse's income is on the return, and at that income (about $115,000) the $190 IRA deduction is fully allowed, which gives AGI of $112,408.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with no derivation. The joint return has $80,208.25 of taxable income (AGI $112,408.25 minus the $32,200 standard deduction), which produces $9,128.99 of tax and $8,628.99 after the $500 ODC."
-us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It added the dependent's $45,000 wages to the joint return's income, which gave AGI of $157,408 and taxable income of $125,208. The joint return counts only the head's and spouse's income, so AGI is $112,408.25 and taxable income is $80,208.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted $16,778 of ESI premiums from wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, used a $30,900 standard deduction, and denied the $500 ODC for the disabled adult child. It then submitted $7,625.75, which matches none of its own intermediate results ($7,248.40 or $9,345)."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It counted all $163,000 of wages, including the dependent's, deducted the full $3,000 of student loan interest, and wrongly asserted that the credit for other dependents is excluded from this metric, although it is a nonrefundable credit that is subtracted. It then submitted $3,778, which contradicts its own computed $13,491 liability."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly got AGI of $112,408 and the $32,200 standard deduction. It then subtracted $1,000 under the new auto loan interest deduction, which requires a qualifying new vehicle assembled in the US and doesn't apply here. That cut taxable income to $79,208 instead of $80,208. It then made an unexplained further cut from $8,509 to $8,067 instead of reporting $9,129 − $500 = $8,629."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the dependent's $45,000 wages to the joint AGI, which pushed taxable income into the 22% bracket. The joint return counts only head and spouse income. It also took a $1,000 auto loan interest deduction that doesn't apply. Correct taxable income is $80,208, giving $9,129 − $500 = $8,629."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's $45,000 wages in AGI and double-counted the retirement deductions. It used an outdated $29,200 standard deduction and pre-2026 brackets instead of $32,200 and the 2026 brackets. It then subtracted an invented $4,246 for 'refundable credits and reconciliation' and never applied the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's own reasoning put the tax at about $5,000, yet it submitted $0. Correct taxable income is $112,408 − $32,200 = $80,208, which gives $9,129 of tax and $8,629 after the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model computed AGI of $112,408 correctly but rejected the 27-year-old as a dependent using the support and gross-income tests. A permanently disabled child is a qualifying child at any age, so the $500 credit for other dependents applies. It also used the 2025 standard deduction of $31,500 and 2025 bracket edges instead of the 2026 values of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the qualifying-relative gross income test to the disabled 27-year-old and so dropped the $500 credit for other dependents. That test doesn't apply to a disabled qualifying child. It also used a $31,500 standard deduction and a $24,000 top for the 10% bracket instead of the 2026 values of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $32,600 standard deduction. It then added about $2,000 for 'the disabled adult dependent's own tax'. The dependent's wages aren't taxed in this tax unit's liability, so its intermediate $8,371 should have been $8,629."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed the joint return includes all three earners' wages and added the dependent's $45,000 to AGI. The joint return counts only head and spouse income. It also deducted $1,000 of auto loan interest that doesn't qualify. Correct taxable income is $80,208, not $124,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the qualifying-relative gross income limit to the disabled 27-year-old and so dropped the $500 credit for other dependents. A permanently disabled child is a qualifying child at any age. It also used the pre-OBBBA $30,000 standard deduction and 2025 brackets instead of $32,200 and the 2026 $24,800 10% bracket."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got AGI of $112,408 right but used a $30,300 standard deduction instead of $32,200, giving a tax of $9,376. It then raised that to $10,800 with no stated computation and never applied the $500 credit for the disabled adult dependent."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied dependent status because of the 27-year-old's $45,000 earnings and so dropped the $500 credit for other dependents. A permanently disabled child is a qualifying child regardless of age or income. It also subtracted a $1,000 auto loan interest deduction that doesn't apply, which lowered taxable income to $79,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's $45,000 wages into the joint AGI and then treated the IRA and student loan deductions as fully phased out because of that inflated income. It used a $30,600 standard deduction and omitted the $500 other-dependent credit. Correct taxable income is $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate single-return tax of $2,354.82 on the dependent's wages to the joint liability, but the reference measures only the joint tax unit. It also subtracted the $8,389 ESI premiums from wages, though they don't reduce AGI here, and didn't net out the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 ESI premiums from each person's wages, cutting joint AGI to $95,630 instead of $112,408. It also folded the dependent's separate single-return income into the total. It never applied the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model added $2,309.98 of tax from a separate single return for the dependent to the joint liability, but only the joint unit's tax counts. It also subtracted ESI premiums from wages, giving joint AGI of $95,630 instead of $112,408."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed TCJA expires in 2026 and applied personal exemptions with a $15,850 standard deduction. OBBBA made the TCJA structure permanent, with a $32,200 standard deduction for married filing jointly. It also subtracted ESI premiums from wages and added a separate tax on the dependent's income."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from household gross income of $163,156, which includes the dependent's $45,000 wages, though the joint return counts only head and spouse income. It also deducted the full $3,000 of student loan interest instead of $2,500, left out the $190 IRA deduction, and skipped the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied pre-TCJA 2026 parameters with personal exemptions. OBBBA permanently kept the TCJA brackets and set a $32,200 standard deduction for married filing jointly. It also added a separate return for the dependent instead of claiming the $500 other-dependent credit on the joint return."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model used post-TCJA-sunset personal exemptions and standard deductions instead of the OBBBA 2026 values of $32,200 and the 10%/12% brackets. It also subtracted ESI premiums to reach AGI of $95,630 and added $3,006.65 of tax from a separate return for the dependent."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The correct derivation gives $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 other-dependent credit. The model's $11,048 is about $2,400 too high, which is consistent with adding a separate tax on the dependent's $45,000 wages to the joint liability."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed the joint return with a separate return for Dependent 1, though only the joint tax unit's liability counts. It also subtracted ESI premiums from wages, though they don't reduce AGI. These two errors roughly offset, but the joint unit alone owes $9,129 − $500 = $8,629."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate single-return tax for Dependent 1 to the joint liability. The dependent is claimed on the joint return, and the output covers only that unit, whose $80,208 of taxable income gives $9,129 − $500 = $8,629."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into household AGI, reaching about $157,598 instead of $112,408. It also applied personal exemptions from the TCJA sunset, which OBBBA prevented. Correct taxable income is $80,208 after the $32,200 standard deduction."
+us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly claimed the $500 other-dependent credit on the joint return. It then also added $3,353.74 of tax from a separate return for the same dependent's wages, which isn't part of the joint unit's liability. It also used a $30,840 standard deduction instead of $32,200."
+us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the dependent as self-supporting and added $3,210 of tax from a separate single return, and so dropped the $500 credit for the disabled qualifying child. It also deducted $3,000 of student loan interest instead of the $2,500 cap."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model claimed the deductions wipe out all tax. In fact, AGI of $112,408 minus the $32,200 standard deduction leaves $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model mixed in itemized medical effects and other unsupported deductions, which put taxable income at roughly $57,000–61,000 instead of $80,208. The medical expenses are far below the 7.5%-of-AGI floor, and the $32,200 standard deduction is the only below-the-line deduction."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in the joint return's box-1 wages and subtracted all three ESI premiums, reaching AGI of $132,241. The correct joint AGI is $112,408, from head and spouse income only, with ESI premiums not reducing wages."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's taxable income of $124,708 includes the dependent's $45,000 wages, which don't belong on the joint return. It also applied no nonrefundable credit, missing the $500 credit for the disabled adult dependent."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's pre-credit tax of $17,011.56 implies about $125,398 of taxable income. That figure includes the dependent's $45,000 wages and leaves out the $190 IRA deduction. The joint unit's taxable income is $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the $500 other-dependent credit correctly. However, its $85,208 of taxable income implies a $27,200 deduction against the $112,408 AGI instead of the 2026 standard deduction of $32,200, which overstated pre-credit tax by $600."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the single household group as one joint return containing the dependent's wages and subtracted all three ESI premiums, reaching AGI of $132,241. The joint unit's AGI is $112,408, from head and spouse income only."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint tax of $9,128.96 matches the engine's. However, it added $3,220 of tax from a separate return for the dependent and left out the $500 credit for the disabled adult dependent claimed on the joint return."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 ESI premiums for both head and spouse from wages, cutting AGI to $95,630. In this computation ESI premiums don't reduce wages, so correct AGI is $112,408, taxable income is $80,208, and the tax is $9,129 before the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $13,980 implies about $111,600 of taxable income on the joint schedule, which is consistent with adding the dependent's wages to the head's and spouse's income. Only head and spouse income counts, which gives $80,208 of taxable income and $8,629 after the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied post-TCJA-sunset rules with personal exemptions and 15% brackets. OBBBA made the TCJA structure permanent, with the $32,200 standard deduction and 10%/12% brackets. It also added a separate $4,846 tax on the dependent's wages instead of claiming the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model used pre-TCJA 2026 parameters: a $16,600 standard deduction, $5,300 exemptions, and a 15% bracket. OBBBA kept the $32,200 standard deduction and the 12% bracket. It also added a separate $4,130 tax on the dependent's wages and omitted the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $3,224 tax on a separate single return for the dependent and omitted the $500 credit for the disabled adult dependent on the joint return. It also deducted $173 of 'overtime premium' even though no overtime pay is listed."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied post-TCJA brackets and a $30,750 standard deduction instead of OBBBA's permanent 2026 values of a $32,200 deduction and a 12% bracket up to $100,800. It also added a separate $3,820 tax on the dependent instead of claiming the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint tax of $9,129 on $80,208 of taxable income is correct. However, it added $3,220 of tax from a separate single return for the dependent and skipped the $500 credit for the disabled adult dependent claimed on the joint return."
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's $45,000 wages into joint AGI. On that inflated income it ruled the $190 IRA deduction fully phased out and also skipped the $4,058 traditional 401(k) exclusion, reaching taxable income of $129,456 instead of $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model reported $0 with no derivation. The joint unit actually has $80,208 of taxable income ($112,408 AGI minus the $32,200 standard deduction), which gives $9,129 of tax and $8,629 after the $500 other-dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model started AGI from $163,000 of wages, which includes the dependent's $45,000, though the joint return counts only head and spouse income. That pushed taxable income to $125,208 and into the 22% bracket instead of $80,208 in the 12% bracket."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted $16,778 of ESI premiums from wages and deducted the full $3,000 of student loan interest instead of the $2,500 cap. It dropped the $190 IRA deduction, denied the $500 credit for the disabled adult dependent, and submitted $7,625.75, which doesn't match its own subtotals."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages for $163,000 of total wages, deducted $3,000 of student loan interest instead of $2,500, and explicitly excluded the $500 other-dependent credit, which is a nonrefundable credit that this output subtracts. It then submitted $3,778, which contradicts its own $13,491 computation."
us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"Its per-person figures were all correct ($3,286/$768.50, $4,030/$942.50, $2,790/$652.50), and they add up to $12,469.50. It summed them to $11,469.50, a $1,000 addition error, and then submitted $12,348.76, which matches neither its own sum nor any calculation it showed."
-us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"It correctly computed $10,106 Social Security plus $2,363.50 Medicare = $12,469.50. It then submitted $12,487.50, so the output field is $18 off from its own correct total."
-us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"It listed the correct 6.2% and 1.45% amounts for all three earners, which add up to $12,469.50. It summed them wrongly to $12,622.90, overstating the total by $153.40."
-us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated each worker's $8,389 of employer-sponsored insurance premiums as a pre-tax Section 125 deduction and cut FICA wages to $137,833. No fact makes those premiums salary-reduction contributions, so the FICA base is the full $163,000 of gross wages, and 7.65% of that is $12,469.50."
-us,scenario_093,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted $8,389 of employer-sponsored insurance premiums from each worker's wages as if they were pre-tax cafeteria-plan deductions, giving a $137,833 base and $10,544.22. The FICA base is the full $163,000 of listed gross wages, and 7.65% of that is $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the right method, 6.2% plus 1.45% of $163,000 in total wages, which equals exactly $12,469.50. It submitted $12,481, an $11.50 arithmetic error."
-us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It explicitly treated the three $8,389 employer-sponsored insurance premiums ($25,167 in total) as Section 125 exclusions and taxed only $137,833. No listed fact designates those premiums as pre-tax salary reductions, so the full $163,000 is taxable and the tax is $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced each worker's FICA wages by $8,389 of employer-sponsored insurance premiums ($44,611, $56,611, $36,611) and got $10,544.22. The premiums are not salary-reduction contributions, so the listed gross wages of $53,000, $65,000 and $45,000 are the FICA base, and the correct total is $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It showed no working. Its $12,654 total implies a 7.65% base of about $165,412, which matches no combination of the listed wages. The correct total is 7.65% of $163,000 = $12,469.50, so its answer is $184.50 too high."
-us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted $8,389 of employer-sponsored insurance premiums from each worker's wages as pre-tax health insurance, taxing only $137,833 and getting $10,544.22. Nothing in the facts makes those premiums a cafeteria-plan salary reduction, so FICA applies to the full $163,000, giving $12,469.50."
-us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly said Social Security and Medicare apply to the full $53,000, $65,000 and $45,000 with no Additional Medicare Tax, which gives $12,469.50. It reported $12,300.30 instead, a $169.20 arithmetic shortfall."
-us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its formula, 0.062 × $163,000 + 0.0145 × $163,000, is correct and equals $10,106 + $2,363.50 = $12,469.50. It evaluated the formula as $9,024, a pure arithmetic error."
-us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It defined FICA wages as gross wages minus pre-tax employer-sponsored insurance premiums ($137,833 in total) and got $10,544.22. The premiums are not designated as Section 125 salary reductions, so the FICA base is the full $163,000 and the tax is $12,469.50."
-us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 7.65% to $137,833, which is wages after subtracting $8,389 per worker of employer-sponsored insurance premiums as pre-tax deductions. No listed fact makes those premiums cafeteria-plan contributions, so FICA applies to the full $163,000 of gross wages, giving $12,469.50."
-us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It netted the employer-plan premiums ($8,389 per worker) out of household wages, got a $137,833 FICA base and reported $10,544.22. The listed gross wages of $163,000 are the payroll-tax base because the premiums are not salary-reduction contributions, so the correct answer is $12,469.50."
-us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value and no explanation, so there is no substantive answer to grade against the $12,469.50 of employee FICA on $163,000 of wages."
-us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"It reported $0 of payroll tax with no working. That ignores the 6.2% Social Security and 1.45% Medicare employee taxes on $163,000 of wages across three earners, which total $12,469.50."
-us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed 7.65% for each earner ($4,054.50 + $4,972.50 + $3,442.50) and itself wrote the total as $12,469.50. It then submitted $12,553.50, so the output field is $84 off from its own correct total."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It deducted the full $3,000 of student loan interest instead of the $2,500 cap, and it skipped Missouri's federal income tax deduction: 5% of the $8,629 federal liability, or $431.45, for MO AGI in the $100,001–$125,000 band. It also ran the graduated schedule once on the joint ~$79,800. Missouri's combined return instead splits taxable income by income percentage ($34,743.94 / $45,032.86) and taxes each spouse's share separately, which uses the low brackets twice and saves about $181."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It put the dependent's wages into the couple's MO AGI ($156,408 instead of the spouses' $112,408.25). That pushed AGI over $125,000 and wrongly eliminated the $431.45 federal-tax deduction. It then taxed the inflated total on one schedule instead of taxing each spouse's income-percentage share of $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from $119,156 of gross income without subtracting the $4,058 of 401(k) deferrals, the $190 IRA deduction or the $2,500 of student loan interest. It used a stale $27,900 standard deduction instead of 2026's $32,200 and replaced Missouri's graduated schedule (4.7% top rate) with an invented flat effective rate of about 4.04%. It also left out the $431.45 federal-tax deduction and the per-spouse split of $79,776.80 of taxable income."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never subtracted the $4,058 of traditional 401(k) deferrals (its AGI of $116,466 = $118,000 + $1,156 − $190 − $2,500). It also used a $30,000 standard deduction instead of $32,200 and skipped the $431.45 federal-tax deduction, which overstated taxable income at $86,466 against the correct $79,776.80. It then applied one schedule to the joint amount instead of taxing each spouse's income-percentage share separately."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It deducted $3,000 of student loan interest instead of the $2,500 cap, used $31,500 instead of the $32,200 standard deduction, and set the federal-tax deduction to zero when it is $431.45 (5% of federal liability in the $100,001–$125,000 MO AGI band). Its own bracket arithmetic came to about $3,640, which it then pushed up to $4,126 with no basis. The correct method splits $79,776.80 between the spouses by income percentage and taxes each share, giving $1,452.33 + $1,935.91."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It added about $1,400 of Missouri tax on a separate return for the dependent's $45,000 of wages. The household is one tax unit with the disabled adult claimed as a dependent, so no dependent tax arises. The couple alone owes $3,388.25 after splitting $79,776.80 by income percentage and taxing each spouse's share."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It folded the dependent's $45,000 of wages into the couple's Missouri AGI ($157,408), which pushed AGI above $125,000 and wiped out the federal-tax deduction. The couple's MO AGI is the spouses' $112,408.25 only. That keeps a $431.45 federal-tax deduction (5% band) and leaves $79,776.80 of taxable income, taxed separately per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted $16,778 of employer-sponsored insurance premiums from wages as if they were pre-tax, took a $4,400 personal exemption that Missouri does not have, and deducted 100% of an estimated $7,339 of federal tax instead of the percentage-limited deduction (5% of $8,629 = $431.45 at MO AGI of $112,408.25). It also deducted the full $3,000 of student loan interest instead of the $2,500 cap, used $30,000 instead of the $32,200 standard deduction, and applied a flat 4.7% instead of the graduated schedule per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It settled on about $102,000 of Missouri taxable income, more than $22,000 above the correct $79,776.80, even though the components it listed net to far less. It skipped the $2,500 student-loan and $190 IRA subtractions, used $30,000 instead of $32,200 for the standard deduction, and left out the $431.45 federal-tax deduction. It also taxed the joint total on one schedule instead of taxing each spouse's income-percentage share."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It got MO AGI ($112,408) and the $32,200 standard deduction right but left out Missouri's federal income tax deduction (5% of the $8,629 federal liability, or $431.45, in the $100,001–$125,000 band), leaving taxable income at $80,208 instead of $79,776.80. It then applied the graduated schedule once to the joint total. Missouri's combined return allocates taxable income by income percentage ($34,743.94 and $45,032.86) and taxes each share separately, which runs the low brackets twice and cuts about $181 from the tax."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It included the dependent's wages in a $160,098 AGI and subtracted $6,300 of personal exemptions that Missouri does not provide. It also used a $30,600 standard deduction instead of $32,200 and a flat 4.5% rate instead of the 2026 graduated schedule with its 4.7% top rate. The correct base is the spouses' $112,408.25 less $32,200 and $431.45, or $79,776.80, taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It added a separate single-filer return for the dependent ($1,056.06), even though the household is one tax unit and the dependent owes no Missouri tax. It also priced the joint return with 2024 brackets and the 2024 $29,200 standard deduction instead of 2026's $32,200. The couple's correct tax is $3,388.25, from taxing each spouse's income-percentage share of $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It totaled $99,241 of taxable income across the couple and a separately taxed dependent, bringing in the dependent's $45,000 of wages, which generate no Missouri tax in this unit. It also applied a 4.6% top rate instead of the 4.7% in Missouri's 2026 schedule. Only the spouses' $79,776.80 is taxable, split by income percentage and taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It cut the couple's AGI to $95,630 by treating $16,778 of employer-sponsored insurance premiums as pre-tax wage reductions. That also moved the couple into the 15% federal-tax-deduction band; the correct MO AGI is $112,408.25, with a 5% deduction of $431.45. It then added a separate $801.81 single-filer tax for the dependent, who belongs to the couple's tax unit and owes no Missouri tax."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It applied a flat 4.8% rate (the 2024 top rate) instead of Missouri's 2026 graduated schedule topping at 4.7%. It also computed tax for both an MFJ unit and a single unit for the dependent, citing both the $10,000 MFJ and $5,000 single federal-tax-deduction caps. Only the couple is taxed: $79,776.80 split between the spouses by income percentage yields $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $6,055 corresponds to about $128,800 of income at 4.7%, which matches taxing the dependent's $45,000 of wages together with the couple's income. The correct computation taxes only the spouses' $79,776.80 ($112,408.25 − $32,200 − $431.45), split by income percentage and taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It taxed the household's combined income, including the dependent's $45,000 of wages, at a rough flat 4.5%. Missouri's 2026 schedule has a 4.7% top rate with graduated lower brackets. Only the spouses' $79,776.80 of taxable income is taxed, split between them by income percentage."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It applied multiple standard deductions and added tax on a separate return for the dependent's $45,000 of wages, a return the one-tax-unit household does not produce. The couple alone owes $3,388.25 after the $32,200 standard deduction, the $431.45 federal-tax deduction and applying the brackets per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $5,321 implies about $117,000 of taxable income at the 4.7% margin, which matches including the dependent's $45,000 of wages. The correct base is the spouses' $79,776.80, split by income percentage and taxed per spouse for $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It named the right components but came out $32.75 high, which at the 4.7% marginal rate equals about $697 of overstated taxable income. The correct base is $112,408.25 − $32,200 standard deduction − $431.45 federal-tax deduction = $79,776.80. Split $34,743.94 / $45,032.86 and taxed per spouse, that gives $1,452.33 + $1,935.91."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It summed Missouri tax over several household tax units, adding tax on the dependent's $45,000 of wages; $4,850 exceeds the couple-only $3,388.25 by $1,461.75. The dependent belongs to the couple's unit and generates no Missouri tax."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $5,250 implies about $115,500 of taxable income at 4.7%, which matches taxing the dependent's $45,000 of wages along with the couple's income. Only the spouses' $79,776.80 is taxed, per spouse, for $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It treated the dependent as a separate single filer and taxed $30,400 of the dependent's income. It used 2024 standard deductions ($29,200 MFJ) instead of 2026's $32,200, skipped the $190 IRA deduction and the $431.45 federal-tax deduction, and applied a flat 4.7% with no lower brackets. The couple's $79,776.80, split by income percentage and taxed per spouse, yields $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It added $1,356 of single-filer tax for the dependent, who belongs to the couple's tax unit and owes no Missouri tax, and applied a flat 4.7% instead of the graduated schedule. It also deducted $3,000 of student loan interest instead of the $2,500 cap and left out the $431.45 federal-tax deduction and the per-spouse split."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and allowances wipe out Missouri taxable income. In fact the couple's $112,408.25 of MO AGI exceeds the $32,200 standard deduction plus the $431.45 federal-tax deduction by $79,776.80, which produces $3,388.25 of tax under the 2026 brackets applied per spouse. No Missouri nonrefundable credit offsets it."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It offered an approximation with no computation. $3,427 is $38.75 above the correct amount, which equals about $824 of overstated taxable income at the 4.7% margin. The correct base is $79,776.80 ($112,408.25 − $32,200 − $431.45), split by income percentage and taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It built a $157,598 AGI that includes the dependent's $45,000 of wages, subtracted $5,400 of personal and dependent exemptions that Missouri does not allow, and used a 4.5% top rate instead of 2026's 4.7%. The spouses' MO AGI is $112,408.25; after the $32,200 standard deduction and the $431.45 federal-tax deduction, $79,776.80 is split between them and taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It reached $124,708 of taxable income by including the dependent's $45,000 of wages in the couple's base. The spouses' MO AGI of $112,408.25 keeps a $431.45 federal-tax deduction and yields $79,776.80 of taxable income. It also applied one schedule to the joint total instead of taxing each spouse's income-percentage share."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It put the couple above the income range for a federal-tax deduction, which only happens if the dependent's $45,000 of wages is folded into AGI. The spouses' MO AGI of $112,408.25 sits in the 5% band ($100,001–$125,000), which gives a $431.45 deduction. Taxing all household income on one schedule inflated the result to $5,720."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted personal exemptions that Missouri does not provide, which put taxable income about $4,166 below the correct $79,776.80 and the tax $195.81 low. The only deductions from the $112,408.25 of MO AGI are the $32,200 standard deduction and the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used $99,041 of Missouri taxable income, $19,264 above the correct $79,776.80. It also allowed no federal-tax deduction, although the spouses' MO AGI of $112,408.25 falls in the 5% band for a $431.45 deduction. It then taxed $99,041 on a single schedule instead of splitting taxable income between the spouses by income percentage."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It taxed only the spouses but understated taxable income by about $5,622, landing $264.25 low. The federal-tax deduction is only 5% of the $8,629 federal liability ($431.45) because MO AGI of $112,408.25 is in the $100,001–$125,000 band. That leaves $79,776.80, taxed per spouse for $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It taxed the couple on $80,208 and added a separate return for the dependent on about $28,900, but the dependent belongs to the couple's tax unit and owes no Missouri tax. The couple's base also leaves out the $431.45 federal-tax deduction, and Missouri taxes each spouse's income-percentage share rather than the joint total."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $61,776.10 of taxable income plus the $32,200 standard deduction and its $1,653.90 federal-tax deduction implies an MO AGI of $95,630. That means it subtracted the $16,778 of employer-sponsored insurance premiums from wages as pre-tax. Without that exclusion, MO AGI is $112,408.25, which drops the federal-tax deduction to the 5% band ($431.45) and leaves $79,776.80, taxed per spouse for $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It stated only that the tax came after the standard deduction; $3,780 corresponds to about $84,300 of taxable income on one schedule. The correct computation subtracts both the $32,200 standard deduction and the $431.45 federal-tax deduction from $112,408.25. It then splits the resulting $79,776.80 by income percentage and taxes each spouse's share separately."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It filed a separate single return for the dependent (about $1,547), overstated the couple's taxable income at $95,365 against $79,776.80, and used a 4.8% top rate instead of 2026's 4.7%. The dependent belongs to the couple's tax unit and owes no Missouri tax, and the couple's tax is $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It added a $1,555 single-filer return for the dependent, who belongs to the couple's tax unit and owes no Missouri tax. It also overstated the couple's taxable income at $95,808 against the correct $79,776.80 and left out the $431.45 federal-tax deduction. It then applied the schedule to the joint total instead of taxing each spouse's income-percentage share."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It taxed the dependent as a separate single filer ($1,196 on $28,900), although the dependent belongs to the couple's tax unit and generates no Missouri tax. For the couple it left out the $431.45 federal-tax deduction and applied the schedule once to $80,035 instead of splitting taxable income by income percentage and taxing each spouse's share."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It used a 4.95% top rate (the 2023 rate) instead of 2026's 4.7%. It added a separate $1,354 return for the dependent, who belongs to the couple's tax unit and owes no Missouri tax. It also skipped the $431.45 federal-tax deduction and the per-spouse split of $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It added a separate single-unit tax of $1,047 for the dependent, but the household is one tax unit and the dependent's wages generate no Missouri tax. The joint piece is $3,388.25: $79,776.80 split by income percentage and taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used obsolete Missouri brackets with 5.3–5.5% rates instead of 2026's schedule topping at 4.7%, and subtracted $6,000 of personal exemptions that Missouri does not allow. It applied the 35% federal-tax-deduction rate (the AGI ≤ $25,000 band) instead of 5% for the $100,001–$125,000 band. It also deducted $3,000 of student loan interest instead of the $2,500 cap and used $29,900 instead of the $32,200 standard deduction, then nudged the result up for 'indexation'."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in the couple's AGI ($161,656), which wrongly eliminated the federal-tax deduction and inflated taxable income to $129,456. The spouses' MO AGI is $112,408.25, giving $79,776.80 of taxable income taxed per spouse for $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 with no reasoning. The couple has $79,776.80 of Missouri taxable income ($112,408.25 MO AGI − $32,200 standard deduction − $431.45 federal-tax deduction) and no Missouri nonrefundable credits, so the tax is $1,452.33 + $1,935.91 = $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It used a $157,408 AGI that includes the dependent's $45,000 of wages and subtracted $3,600 of exemptions that Missouri does not provide. The spouses' MO AGI of $112,408.25 keeps a $431.45 federal-tax deduction and yields $79,776.80 of taxable income, taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in AGI ($164,156) and skipped the 401(k), IRA and student-loan subtractions. It used a $26,600 standard deduction instead of $32,200 and assumed a 4.95% rate. It then reported $11,145.20, which contradicts its own $6,809 calculation; the correct base is the spouses' $79,776.80, taxed per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"Its $124,942 starting point equals $157,142 − $32,200, which means the dependent's $45,000 of wages was folded into the couple's base. It then applied a 4.5% rate instead of the 2026 schedule topping at 4.7%. Only the spouses' $79,776.80 is taxable, after a $431.45 federal-tax deduction that its inflated AGI eliminated, and it is taxed per spouse."
+us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It got all six per-person components right ($3,286, $768.50, $4,030, $942.50, $2,790, $652.50), but added them to $11,469.50 instead of $12,469.50. It then submitted $12,348.76, a figure that matches neither its own sum nor its components."
+us,scenario_093,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its reasoning names the correct method, 7.65% employee FICA on $163,000 of combined wages with no Additional Medicare Tax. It never did the multiplication, which gives $12,469.50, and submitted 0 instead."
+us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"It correctly computed $10,106 Social Security plus $2,363.50 Medicare for a total of $12,469.50. It then submitted $12,487.50, so the error was in copying its own total into the answer."
+us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"Each per-person Social Security and Medicare figure was correct. However, it added the six amounts to $12,622.90 instead of the correct $12,469.50."
+us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It subtracted $8,389 of ESI premiums from each worker's wages as if they were pre-tax Section 125 deductions, giving $137,833 of FICA wages. The facts give only gross wages and never describe a pre-tax salary reduction, so the full $163,000 is FICA wages and the tax is $12,469.50."
+us,scenario_093,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It assumed the $8,389 ESI premium came out of each worker's pay pre-tax and lowered the FICA base to $137,833. No such cafeteria-plan reduction is in the facts. Applying 7.65% to the full $163,000 of gross wages gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the correct base ($163,000) and rate (6.2% + 1.45%) but got the multiplication wrong. 7.65% × $163,000 is $12,469.50, not $12,481."
+us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the three $8,389 ESI premiums ($25,167 total) as Section 125 exclusions and reduced FICA wages to $137,833. The household facts list only gross wages and no pre-tax premium deduction, so FICA applies to the full $163,000, which gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It set each worker's FICA wages to gross pay minus the $8,389 ESI premium ($44,611, $56,611, $36,611), which assumes a pre-tax payroll deduction that isn't in the facts. Using the full gross wages of $53,000, $65,000 and $45,000 gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $12,654 with no derivation, which is $184.50 too high. The correct calculation is 6.2% + 1.45% on $163,000 of gross wages, all under the wage base and with no Additional Medicare Tax, which gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It deducted the $8,389 ESI premium from each worker's wages as a pre-tax health deduction and taxed only $137,833. The facts specify gross wages with no cafeteria-plan reduction, so all $163,000 is subject to FICA, which gives $12,469.50."
+us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly applied Social Security and Medicare to the full $53,000, $65,000 and $45,000 with no Additional Medicare Tax. But it calculated $12,300.30 instead of 7.65% × $163,000 = $12,469.50."
+us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its formula, 0.062 × $163,000 + 0.0145 × $163,000, is correct. It got the arithmetic wrong, though: $10,106 + $2,363.50 = $12,469.50, not $9,024."
+us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It treated each $8,389 ESI premium as a pre-tax deduction from FICA wages, cutting the base to $137,833. The facts give gross wages with no pre-tax premium reduction, so 7.65% of the full $163,000 is $12,469.50."
+us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It calculated FICA on $137,833 by assuming the ESI premiums were pre-tax payroll deductions. No cafeteria-plan salary reduction is listed, so the base is the full $163,000 of gross wages, which gives $12,469.50."
+us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It subtracted the employer-plan premiums from wages before applying 7.65%, which assumes a Section 125 exclusion that isn't in the facts. FICA applies to the full $163,000 of gross wages, which gives $12,469.50."
+us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation came back, so no answer could be scored."
+us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"It submitted 0 with no reasoning, which means no employee FICA at all on $163,000 of W-2 wages. All three workers owe 6.2% Social Security plus 1.45% Medicare, for a total of $12,469.50."
+us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $4,054.50 + $4,972.50 + $3,442.50 = $12,469.50. It then submitted $12,553.50, so the error was in copying its own total into the answer."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got roughly the right taxable base (~$79,800) but applied Missouri's schedule only once to the joint total. It should have split taxable income between the spouses by income share and taxed each share separately, which gives the lower-bracket savings twice. It also left out the 5% federal-tax deduction ($431.45) and deducted the full $3,000 of student loan interest instead of the $2,500 cap."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the dependent's wages in joint Missouri AGI ($156,408 instead of $112,408.25). That pushed AGI above $125,000 and wrongly removed the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's income-proportional share."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model used gross income of $119,156 without excluding 401(k) deferrals, IRA contributions or student loan interest. It used a $27,900 standard deduction instead of the 2026 MFJ $32,200. It then multiplied by a made-up 4.04% effective rate instead of applying Missouri's bracket schedule to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model answered $0 with no working. The spouses' $112,408.25 of Missouri AGI, minus the $32,200 standard deduction and $431.45 federal-tax deduction, leaves $79,776.80 of taxable income. Taxing each spouse's share at rates up to 4.7% gives $3,388.25. The zero answer treats the couple as owing no Missouri tax."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model listed the $4,058 of 401(k) deferrals but left them out of its AGI arithmetic, overstating AGI at $116,466. It used a ~$30,000 standard deduction instead of $32,200 and skipped the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model's own formula gave about $3,640 on $80,408 of taxable income, but it then inflated that to $4,126 with no basis. It also used a $31,500 standard deduction instead of $32,200, deducted $3,000 of student loan interest instead of the $2,500 cap, and dropped the 5% federal-tax deduction as 'near zero'. It never split taxable income between the spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate Missouri tax for the dependent's $45,000 of wages, but the dependent is in the couple's tax unit with $0 Missouri AGI and owes nothing. Its couple-only figure (~$3,400) was close, and the extra dependent tax pushed the total to $4,180."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into joint Missouri AGI ($157,408 instead of $112,408.25). That inflated taxable income to $125,208 and wrongly eliminated the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 employer-sponsored insurance premiums from each spouse's wages, understating AGI at $95,130. It added a $4,400 personal exemption, which Missouri no longer allows. It deducted 100% of federal tax instead of the AGI-based percentage, and it applied a flat 4.7% instead of the graduated schedule on each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model estimated Missouri taxable income at ~$102,000, even though $112,408.25 of AGI minus the $32,200 standard deduction and the $431.45 federal-tax deduction gives $79,776.80. It then applied the top rate to that overstated base without splitting income between the spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got Missouri AGI ($112,408) and the $32,200 standard deduction right, but it omitted the 5% federal-tax deduction ($431.45) for MO AGI of $100,001–$125,000. It then applied the schedule once to the joint $80,208. Missouri splits taxable income between the spouses by income share and taxes each share separately, which roughly doubles the lower-bracket savings."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in a $160,098 AGI and subtracted $2,100 personal exemptions for three people, which Missouri does not allow. It used a $30,600 standard deduction instead of $32,200 and a flat 4.5% instead of the graduated schedule with a 4.7% top rate."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,056.06 single-filer Missouri tax for the dependent, but the dependent is in the couple's unit with $0 Missouri AGI. It also used 2024 brackets and standard deductions instead of the 2026 $32,200 deduction and published 2026 schedule."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $99,241 of total taxable income includes the dependent's wages, which carry $0 Missouri AGI in this unit. It also used a 4.6% top rate instead of 4.7%, and it never split the couple's taxable income between the spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint AGI of $95,630 is exactly $16,778 too low because it subtracted both spouses' employer-sponsored insurance premiums from wages. It then added a separate $801.81 tax for the dependent, who is in the couple's unit with $0 Missouri AGI."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a 4.8% top rate instead of 4.7%. It treated the federal-tax deduction as limited only by the $10,000/$5,000 caps, when at MO AGI of $100,001–$125,000 only 5% of federal tax ($431.45) is deductible. It also computed a separate single return for the dependent."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $6,055 is about $2,670 above the $3,388.25 from taxing only the spouses' $79,776.80, which matches about $57,000 of extra income taxed at 4.7%. That pattern fits taxing the dependent's $45,000 of wages, losing the federal-tax deduction, and skipping the per-spouse split."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model estimated a 'combined' household tax at a flat ~4.5% rate, which taxes the dependent's wages even though the dependent has $0 Missouri AGI. It also never applied the 4.7%-top-rate graduated schedule to each spouse's share of the couple's $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $4,166 is about $780 above the $3,388.25 owed on the spouses' income alone. That gap matches an extra Missouri tax on the dependent's wages, but the dependent is in the couple's unit with $0 Missouri AGI."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $5,321 is about $1,930 above $3,388.25, which matches roughly $41,000 more income taxed at 4.7%. That means the dependent's $45,000 of wages were taxed, though the dependent has $0 Missouri AGI here."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,421 is exactly 4.7% × $700 above the correct $3,388.25. This means it used the 2025 MFJ standard deduction of $31,500 instead of the 2026 amount of $32,200, while otherwise following the per-spouse computation."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed tax across 'household tax units', treating the dependent as a separate taxpayer on $45,000 of wages. The dependent belongs to the couple's unit with $0 Missouri AGI, and only the spouses' $79,776.80 of taxable income is taxed."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $5,250 is about $1,860 above $3,388.25, which matches roughly $40,000 more income taxed at 4.7%. That means the dependent's wages were taxed, though the engine gives the dependent $0 Missouri AGI."
+us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the dependent on a separate single return, but the dependent has $0 Missouri AGI in the couple's unit. It used 2024 standard deductions ($29,200/$14,600) instead of $32,200, omitted the 5% federal-tax deduction, and applied a flat 4.7% instead of the graduated schedule on each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,356 tax for the dependent and applied a flat 4.7% to the couple's $79,708 with no graduated lower brackets. It omitted the 5% federal-tax deduction and deducted $3,000 of student loan interest instead of the $2,500 cap."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model claimed the standard deduction fully offsets the couple's income. But $112,408.25 of Missouri AGI minus the $32,200 deduction and $431.45 federal-tax deduction still leaves $79,776.80 taxable, which produces $3,388.25 of tax."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave an unanchored estimate of 'typical' figures instead of computing the tax. It never did the required steps: subtract the 5% federal-tax deduction ($431.45), split $79,776.80 between the spouses ($34,743.94 and $45,032.86), and tax each share separately for $1,452.33 + $1,935.91."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in a $157,598 federal AGI and subtracted $5,400 of Missouri personal and dependent exemptions, which Missouri no longer allows. It also used a 4.5% top rate instead of the 4.7% top rate in the 2026 schedule."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $124,708 of taxable income includes the dependent's ~$44,500 of wages in joint income, though the dependent has $0 Missouri AGI. With the inflated AGI it took no federal-tax deduction, and it applied the schedule once instead of to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model concluded income was above the federal-tax-deduction range, which only happens if the dependent's wages are added. The spouses' Missouri AGI of $112,408.25 qualifies for the 5% federal-tax deduction ($431.45), and only their $79,776.80 is taxed."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted Missouri personal exemptions, which Missouri no longer allows. That and an oversized federal-tax deduction cut taxable income about $4,200 below the correct $79,776.80, where the only allowed deductions are the $32,200 standard deduction and 5% of federal tax ($431.45)."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $99,041 of taxable income implies an AGI of about $132,000, above the $125,000 cutoff, so it wrongly took no federal-tax deduction. The spouses' AGI is $112,408.25, which qualifies for the 5% deduction. It also applied the schedule once (4.7% × $99,041 − ~$180) instead of to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,124 is $264 below the correct figure, which equals about $5,600 too little taxable income. That fits a federal-tax deduction far larger than the 5% of federal tax ($431.45) allowed at MO AGI of $100,001–$125,000."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the 'working adult' dependent on $28,900 of taxable income, though the dependent has $0 Missouri AGI in the couple's unit. It also gave the spouses $80,208 of taxable income by omitting the 5% federal-tax deduction ($431.45), and it skipped the per-spouse split."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's implied Missouri AGI of $95,630 is $16,778 too low because it subtracted both spouses' $8,389 employer-sponsored insurance premiums from wages. The lower AGI also moved it into the 15% federal-tax deduction tier ($1,653.90) instead of the 5% tier ($431.45)."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,780 equals applying the schedule once to about $84,277 of joint taxable income (4.7% × income − ~$181). That base is $4,500 above $79,776.80, consistent with using the outdated $27,700 standard deduction. Missouri's per-spouse split of $79,776.80, with each share taxed separately, gives $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's couple taxable income of $95,365 reflects subtracting only a single-filer $16,100 standard deduction instead of the MFJ $32,200. It added a separate tax for the dependent, who has $0 Missouri AGI, and used a 4.8% top rate instead of 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model's MFJ taxable income of $95,808 is $111,908 minus a single-filer $16,100 standard deduction, when the MFJ deduction is $32,200. It also added a $1,555 tax for the dependent, who has $0 Missouri AGI in the couple's unit."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,196 single-return tax for the dependent, who carries $0 Missouri AGI in the couple's unit. For the spouses it omitted the 5% federal-tax deduction and applied the schedule once to $80,035 instead of to each spouse's income-proportional share."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,354 tax for the dependent, who has $0 Missouri AGI. It also used the outdated 4.95% top rate instead of 4.7% and ignored the 5% federal-tax deduction and the per-spouse split."
+us,scenario_093,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,047 single-unit tax for the dependent, but the dependent is in the couple's tax unit with $0 Missouri AGI. Only the spouses' $79,776.80 of taxable income is taxed, split by income share, which gives $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model used invented Missouri parameters: a $13,000 or $29,900 standard deduction instead of $32,200, $2,000 personal exemptions that Missouri no longer allows, and a 5.3% rate. It applied the 35% federal-tax-deduction tier, which only covers AGI up to $25,000, instead of the 5% tier for $100,001–$125,000, then raised the result by an arbitrary 'indexation'."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from a $161,656 household AGI that includes the dependent's wages, which pushed income past $125,000 and wrongly eliminated the federal-tax deduction. It then applied the schedule once to $129,456 instead of to each spouse's share of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model answered $0 with no working. The spouses' $79,776.80 of Missouri taxable income, split $34,743.94 / $45,032.86 and taxed separately, produces $3,388.25. The zero answer ignores that Missouri tax is owed above the $32,200 standard deduction."
+us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into a $157,408 joint AGI and subtracted $3,600 of personal exemptions, which Missouri does not allow. It also skipped the 5% federal-tax deduction available at the spouses' true $112,408.25 AGI and applied the schedule once instead of per spouse."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the dependent's $45,000 to joint income, skipped 401(k), IRA and student-loan exclusions, and used a $26,600 standard deduction and a 4.95% rate. It then reported $11,145.20, which contradicts its own $6,809 estimate."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model's ~$124,942 base includes the dependent's wages, though only the spouses' income counts, giving $79,776.80 taxable. It also used a 4.5% top rate instead of the 4.7% top rate in the 2026 schedule."
us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI."
us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero."
us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets."
@@ -7645,12 +7838,13 @@ us,scenario_095,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu
us,scenario_095,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model treated self-employment income as subject to employee-side payroll tax and placed an estimated SECA amount in payroll_tax. Because no wages are listed, the employee Social Security and Medicare tax base is zero; any tax on the $4,198 belongs exclusively in self_employment_tax."
us,scenario_095,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly applied the employee Social Security and Medicare rates to self-employment income, which belongs in the separate self_employment_tax calculation. It also submitted $1,167.84 despite deriving $321.15 in its explanation, but both figures wrongly assign self-employment tax components to payroll_tax instead of using the zero wage base."
us,scenario_095,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_095,self_employment_tax,claude-fable-5,llm_error,other,False,"The model derived the correct chain in its explanation — $4,198 × 0.9235 = $3,876.85 net earnings, clearing the $400 floor, × 15.3% = $593.16 — and then submitted 594.29, a figure $1.13 above the result it had just computed and produced by no step it showed. The substantive rule application was right; the model failed to transcribe its own computed value into the numeric field, violating the requirement that the submitted value match the explanation's stated amount."
-us,scenario_095,self_employment_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model inverted the §1402(b) de minimis test, asserting that $4,198 of self-employment earnings falls 'below the $400 threshold' when $4,198 exceeds $400 by more than tenfold. The threshold is a floor that exempts net earnings under $400, not a ceiling, so net earnings of $3,876.85 are fully subject to the 15.3% combined rate rather than producing zero."
-us,scenario_095,self_employment_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model identified the right structure — net earnings times the combined 15.3% rate — but rounded the statutory 92.35% net-earnings factor of §1402(a)(12) down to 92%: $4,198 × 0.92 × 0.153 = $590.91, which is exactly the ~$591 it submitted. Carrying the exact 0.9235 factor gives the $3,876.85 base that yields the reference $593.23."
-us,scenario_095,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the 15.3% rate to gross self-employment income ($4,198 × 0.153 = $642.29 by its own arithmetic), skipping the §1402(a)(12) reduction of the base to 92.35% of net profit, and then added roughly $10 more in the name of a 'half SE tax deduction effect' that would move the number down, not up, if it applied to SE tax at all. Reducing the base to $3,876.85 before applying 15.3% is the step it omitted, and the half-of-SE-tax deduction is an income-tax adjustment that never enters the SE tax computation."
-us,scenario_095,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for self_employment_tax, so the required key was absent from the submitted outputs object. This is a contract failure in producing the answer rather than a misapplication of the §1401 rate or the 92.35% net-earnings base."
-us,scenario_095,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly noted that $4,198 clears the $400 floor and then zeroed the liability by treating 'the self-employment tax deduction and other offsets' as reductions of the SE tax itself. The §164(f) deduction for one-half of SE tax reduces adjusted gross income for income-tax purposes only; §1401 liability of 15.3% on the $3,876.85 net-earnings base stands in full regardless of that deduction, the taxpayer's age, or Social Security receipt."
+us,scenario_095,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model applied the method correctly and got 4,198 × 0.9235 × 0.153 ≈ $593.16, but then submitted $594.29. That number does not follow from its own arithmetic. The final answer was a transcription error, not a mistake in method."
+us,scenario_095,self_employment_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model correctly found net SE earnings of about $3,877 and tax of about $593. It then added an unexplained adjustment for 'estimated rounding of SE base' and submitted $598. No rule supports that adjustment, and the extra $5 is wrong."
+us,scenario_095,self_employment_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model claimed that $4,198 of self-employment income is below the $400 SE-tax threshold, which is a basic comparison error. Net SE earnings of about $3,877 (4,198 × 0.9235) are far above $400, so the full 15.3% SE tax of about $593 applies."
+us,scenario_095,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model knew the 15.3% rate applies but never worked out the exact base. It gave a rough estimate of about $591 when the actual calculation is 4,198 × 0.9235 = $3,877 × 0.153 ≈ $593. Using the exact 0.9235 net-earnings factor removes the $2 error."
+us,scenario_095,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied 15.3% to the full $4,198 and skipped the step that reduces income to net earnings by multiplying by 0.9235. It then made an unexplained jump from its own $642 to $652.45. It also wrongly said the half-SE-tax deduction raises the SE tax, when that deduction only lowers income-tax AGI."
+us,scenario_095,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no self_employment_tax value and no explanation, so there was nothing to score against the required amount of about $593 (net SE earnings of $3,877 × 15.3%)."
+us,scenario_095,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model said the self-employment tax deduction and other offsets reduce SE tax to zero. The deduction for half of SE tax only reduces adjusted gross income for income tax. It does not reduce SE tax itself. Net SE earnings of about $3,877 are above $400, so SE tax of about $593 is owed whatever the head's age or Social Security receipt."
us,scenario_095,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly exempted this elderly household from the applicable gross-income eligibility screen and proceeded directly to net-income deductions. Its own estimated 30% contribution of about $660 per month exceeds the roughly $292 maximum allotment, which yields $0 rather than the asserted $2,363 annual benefit."
us,scenario_095,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model incorrectly routed the household through an elderly net-income-only pathway after counting $29,605 of annual gross income. That income fails the applicable gross-income eligibility limit, so neither medical and earned-income deductions nor a minimum allotment can create the asserted $300 benefit."
us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an incorrect 200%-of-FPL broad-based categorical eligibility threshold and then invented a $95 monthly New Jersey minimum SNAP benefit. The household’s $29,605 income exceeds the applicable gross-income limit, and the minimum allotment applies only to an otherwise eligible household."
@@ -7724,141 +7918,143 @@ us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi
us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible."
us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility."
us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at the old 20% rate ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It deducted $1,575 of auto-loan interest although no qualifying new, U.S.-assembled vehicle is listed, and it used a $32,600 standard deduction instead of $32,200. Its own arithmetic reached $10,965, yet it submitted $12,103, which none of its steps support."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its $32,200 standard deduction and $6,320 of credits ($4,400 CTC + $1,920 CDCC) were right. It left out OBBBA's $2,000 joint deduction for non-itemizers' cash gifts, which the couple's $11,933 of cash donations supports. It also capped the $338 educator expense at $300, so taxable income was $127,422 instead of $125,385.05, overstating tax by about $448."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It built AGI of $165,502 without removing the $3,088 of traditional 401(k) deferrals from wages or taking the $3,000 net capital loss deduction. It used the 2024 $29,200 standard deduction instead of 2026's $32,200 plus the $2,000 non-itemizer charitable deduction. It then subtracted a made-up $3,174 CTC with no CDCC, instead of the full $4,400 CTC and a $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-OBBBA credit amounts: a $2,000-per-child CTC instead of $2,200 and a 20% CDCC instead of the 32% rate at this AGI, giving $5,200 of credits instead of $6,320. It also used a $31,500 standard deduction and made-up bracket thresholds ($24,150/$98,150) instead of $32,200 and $24,800/$100,800, and it left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its own work ended at $12,421, but it submitted $18,962, which is more than its own $17,621 pre-credit tax, so the submitted value subtracts no credits. The work itself counted the $128 of qualified dividends twice and used a $31,500 standard deduction without the $2,000 non-itemizer charitable deduction. It also used a $2,000-per-child CTC and a 20% CDCC instead of $2,200 and 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put tax at about $19,650 on roughly $127,200 of taxable income, about $2,200 more than the 2026 brackets (10% to $24,800, 12% to $100,800, then 22%) produce. It then subtracted only a $4,000 CTC ($2,000 per child instead of $2,200) and zeroed out the CDCC. With two working parents and $12,740 of childcare for two children, the CDCC is $1,920 (32% of $6,000). It also left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its credits ($4,400 CTC and $1,920 CDCC) and bracket method were right. However, it deducted $1,575 of auto-loan interest with no listed fact showing a qualifying new, U.S.-assembled vehicle, and it left out the $2,000 non-itemizer charitable deduction that the $11,933 of cash gifts supports. With the $300 educator cap, taxable income was $125,847 instead of $125,385.05, adding $101.63 of tax."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It treated only $300 per child ($600) of the CTC as nonrefundable. In fact the whole CTC is applied against tax first, and only the unused remainder goes to the refundable ACTC, so with about $17,000 of tax the full $4,400 ($2,200 per child under OBBBA) is used. It also took the CDCC at 20% instead of 32% and used 2025 bracket thresholds. Its standard deduction was $30,000 instead of $32,200, and it had no $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It estimated about $19,900 of tax before credits. Instead of subtracting credits, it submitted $24,500, citing an AMT add-back that does not apply to someone taking the standard deduction at this income. The correct steps are the $32,200 standard deduction plus the $2,000 charitable deduction (not its $30,300), $16,999.75 of tax, then subtracting the $4,400 CTC and $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also swapped the $2,000 non-itemizer charitable deduction for a $1,575 auto-loan interest deduction that no qualifying-vehicle fact supports, and it disallowed the IRA deduction. Together these overstate the result by about $853."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It counted only $600 of the CTC as nonrefundable, but the full $4,400 ($2,200 per child) is applied against its roughly $17,000 of tax before any refundable portion is figured. It dropped the $7,018 long-term capital loss, missing the $3,000 net capital loss deduction. It also used a 20% CDCC, a $30,900 standard deduction, and no $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied expired pre-TCJA law: four $5,061 personal exemptions, itemizing, 15/25% brackets, and a $1,000-per-child CTC reduced to $750. OBBBA made TCJA permanent, so 2026 has no personal exemptions, a $32,200 standard deduction, 10/12/22% brackets, and a $2,200-per-child CTC. It also subtracted the $24,597 of employer-paid health premiums from wages, putting AGI at $134,987 instead of $159,585.05; its closeness to the right answer comes from errors that cancel out."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used TCJA-sunset rules ($21,200 of personal exemptions, itemized deductions, and a phased-down CTC) instead of OBBBA's permanent 2026 rules: a $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and a $2,200-per-child CTC. It also treated employer-paid health premiums as a pre-tax wage reduction, cutting AGI to $135,069 instead of $159,585.05, and it took the CDCC at 20% instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,700 standard deduction instead of 2026's $32,200 and left out the $2,000 non-itemizer charitable deduction. It then subtracted a $2,000-per-child CTC ($4,000) and a 20% CDCC ($1,200) instead of OBBBA's $2,200 per child and 32% rate ($6,320 total), overstating the result by about $2,088."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and applied $20,200 of personal exemptions, itemized deductions, and a $1,000-per-child CTC. OBBBA made the TCJA framework permanent, so 2026 uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, no exemptions, a $2,200-per-child CTC, and a 32% CDCC ($1,920), not 20%."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It reported AGI of $164,481 without the $3,000 net capital loss deduction. Its $17,822 is essentially tax before credits: it never subtracted the $4,400 CTC and $1,920 CDCC that bring $16,999.75 down to $10,679.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied the TCJA sunset ($21,200 of personal exemptions, itemizing, and a $1,000 CTC reduced to $700), although OBBBA made TCJA permanent with a $32,200 standard deduction and a $2,200-per-child CTC that phases out only above $400,000. It subtracted employer-paid health premiums from wages, getting AGI of $135,169 instead of $159,585.05, and it took the CDCC at 20% instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules ($20,200 of personal exemptions, itemizing, and a $1,000 CTC reduced to $750) instead of OBBBA's permanent rules: a $32,200 standard deduction plus the $2,000 charitable deduction and a $2,200-per-child CTC. It also reduced AGI to about $135,000 by subtracting employer-paid health premiums, and it took the CDCC at 20% instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working; the correct result is $16,999.75 of tax on $125,385.05 of taxable income minus the $4,400 CTC and $1,920 CDCC. Its $11,463 is $783.25 too high, which matches taking the CDCC at the pre-OBBBA 20% rate ($1,200 instead of $1,920, +$720) plus about $290 of extra taxable income."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $11,105 is $425.25 above the correct $10,679.75, which is 22% of about $1,933. That matches applying the full $6,320 of credits while leaving out OBBBA's $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It computed tax after itemized deductions and personal exemptions, which is expired pre-TCJA law. For 2026, OBBBA permanently repealed personal exemptions, and the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction beats itemizing. The credits are the $2,200-per-child CTC and the 32% CDCC, giving $16,999.75 minus $6,320."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $10,817 is $137.25 above the correct $10,679.75, which equals about $624 of extra taxable income with the correct $6,320 of credits. That matches small deduction slips such as disallowing the $144 IRA deduction, capping the educator expense at $300, and swapping auto-loan interest for the $2,000 charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It made up a $15,600 nonrefundable CTC; the 2026 CTC is $2,200 × 2 = $4,400. It limited the CDCC to one child's $3,000 at 20%, although two children qualify for the $6,000 cap at 32%. It claimed a $1,404 saver's credit even though joint AGI near $160,000 is far above that credit's limit, and it never removed the 401(k) deferrals or took the capital loss deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only an $800 nonrefundable portion of the CTC, but the full $4,400 ($2,200 per child) is applied against its roughly $17,500 of tax before any refundable portion is figured. It also took the CDCC at 20% instead of 32% and left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. Its $17,278 is higher than the correct $16,999.75 of tax before credits, so in effect it subtracted none of the $4,400 CTC and $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. Its $14,696 implies only about $2,300 of credits subtracted from roughly $17,000 of tax, instead of the full $4,400 CTC plus $1,920 CDCC ($6,320)."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also left out the $2,000 non-itemizer charitable deduction, so taxable income was $127,566 instead of $125,385.05. These two misses make up its $1,200 overstatement."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $1,200 CDCC (20% of $6,000) instead of $1,920. Under OBBBA the rate is 32% at $159,585 joint AGI: 35% minus one point per $4,000 over $150,000. That overstates the result by $720, partly offset by tax before credits about $211 below the correct $16,999.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% ($1,920). It also left out the $2,000 non-itemizer charitable deduction and the $144 IRA deduction, so taxable income was $127,528 instead of $125,385.05."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It used a $2,000-per-child CTC ($4,000) instead of OBBBA's $2,200 ($4,400) and a 20% CDCC ($1,200) instead of 32% ($1,920). Its roughly $128,000 of taxable income also leaves out the $2,000 non-itemizer charitable deduction that brings taxable income to $125,385.05."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Everything else matched, but it disallowed the $144 traditional IRA deduction. That put AGI at $159,728 instead of $159,585.05, which reflects about $482 of adjustments (the $338 educator expense plus the $144 IRA contributions). The extra $143 of taxable income taxed at 22% is its $31.45 overstatement."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its tax of about $18,360 on $127,566 of taxable income is roughly $880 more than the 2026 brackets (10% to $24,800, 12% to $100,800, then 22%) produce. It also left out the $2,000 non-itemizer charitable deduction and took the CDCC at 20% ($1,200) instead of 32% ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out OBBBA's $2,000 non-itemizer charitable deduction, so taxable income was $127,384 instead of $125,385.05. It also took the CDCC at 20% ($1,200) instead of the 32% rate that applies at $159,585 joint AGI ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It cut the OBBBA CDCC rate by one point per $2,000 over $150,000, reaching 30% ($1,800). For joint returns the step is $4,000, which gives 32% ($1,920). It also disallowed the $144 traditional IRA deduction, adding $31.45 of tax."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Its rough $14,000 guess implies only about $3,000 of credits subtracted from the $16,999.75 of tax before credits. The full $4,400 CTC ($2,200 × 2) and $1,920 CDCC (32% of $6,000) apply, giving $10,679.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law: $21,200 of personal exemptions, 2017-style brackets, a 10% medical floor, and a fully phased-out $1,000 CTC. OBBBA permanently kept the TCJA rates and set the CTC at $2,200 per child with a $400,000 joint phase-out threshold. It then subtracted only a 20% CDCC ($1,200) instead of the $4,400 CTC plus a 32% CDCC ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as after the TCJA sunset: personal exemptions, a $16,600 standard deduction, 10/15/25% brackets, and a CTC phased out above $110,000. OBBBA made the TCJA framework permanent, with a $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, 10/12/22% brackets, and a $2,200-per-child CTC that is untouched below $400,000. It also took the CDCC at 20% instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA law: four $5,400 personal exemptions, itemizing, 10/15/25% brackets, and a CTC fully phased out above $110,000. OBBBA made TCJA permanent, so 2026 uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and a $2,200-per-child CTC ($4,400). It also used a 20% CDCC instead of 32% ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income on 10/15/25% pre-TCJA brackets with a $1,000-per-child CTC reduced to $751, although OBBBA made the TCJA brackets and the $2,200-per-child CTC permanent. It also subtracted employer-paid health premiums from wages, putting AGI at $134,987 instead of $159,585.05, and it took the CDCC at 20% instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also used the $32,200 standard deduction without the extra $2,000 non-itemizer charitable deduction, so its tax before credits of about $17,508 is about $500 too high."
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is nothing to score. The correct figure is $16,999.75 of tax minus $6,320 of credits."
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is nothing to score. The correct figure is $16,999.75 of tax minus $6,320 of credits."
-us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It assumed charitable and other itemized deductions wipe out the tax. Those deductions ($8,654 of property tax plus CA income tax, $12,621 of gifts less the 0.5%-of-AGI floor, and medical costs below the 7.5% floor) fall short of the $32,200 standard deduction. Taxable income is $125,385.05 with $16,999.75 of tax, leaving $10,679.75 after $6,320 of credits."
-us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% ($1,920). It also used 2025 bracket thresholds ($23,850/$96,950) instead of 2026's $24,800/$100,800 and left out the $2,000 non-itemizer charitable deduction, overstating tax before credits by about $850."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached $11,730, but it submitted $24,711.50, which no step supports. The work itself subtracted the whole $6,478 net capital loss instead of the $3,000 limit and used a $2,000-per-child CTC and a 20% CDCC. It also used a $32,600 standard deduction without the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $4,400 child tax credit and applied only a $1,200 CDCC. It also made up a $4,160 educator expense (the actual figure is $338) and skipped the $3,000 capital loss deduction. Its $29,994 standard deduction should have been $32,200 plus the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed the CDCC at the old 20% rate ($1,200) instead of the OBBBA 32% rate for joint AGI of about $159.7k ($1,920). It also deducted the $144 IRA (not deductible because both spouses have workplace plans) and $1,575 of auto-loan interest (no qualifying new vehicle), and used a $32,600 standard deduction. Its submitted $12,103 does not even match its own computed $10,965."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took only the $32,200 standard deduction and left out the OBBBA $2,000 charitable deduction for non-itemizing joint filers, which overstated taxable income by about $1,900. It also took a $144 traditional IRA deduction that the active-participant phaseout eliminates. Its credits ($6,320) were correct, so the whole error is in taxable income."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built AGI of $165,502 without removing the $3,087 of traditional 401(k) deferrals or applying the $3,000 capital-loss deduction, and used the 2024 $29,200 standard deduction. It then subtracted an invented $3,174 CTC and no CDCC, instead of the $4,400 CTC plus the $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rough $6,900 with no bracket calculation. The correct path is $17,031 of regular tax on $125,529 taxable income, minus the $4,400 CTC and $1,920 CDCC, which gives $10,711. Its answer understates tax by about $3,800, consistent with guessing at taxable income instead of applying the 2026 MFJ brackets."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used pre-OBBBA parameters: a $31,500 standard deduction, a $2,000-per-child CTC instead of $2,200, and a flat 20% CDCC instead of the 32% enhanced rate. It also left out the $2,000 non-itemizer charitable deduction and took the IRA deduction, which the active-participant phaseout eliminates. Its credits ($5,200) fell $1,120 short of $6,320."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It counted the $128 of qualified dividends twice in income and used a $31,500 standard deduction with no $2,000 charitable deduction. It also used a $2,000-per-child CTC and a 20% CDCC. It then submitted $18,962, which ignores its own net figure of $12,421 after credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It set the CDCC to about $0 even though $6,000 of qualifying expenses at the 32% rate gives $1,920. It used a $4,000 CTC instead of $4,400 and overstated pre-credit tax at $19,650 on about $127k of taxable income (the correct figure is about $17.0k). It also left out the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $1,575 of auto-loan interest even though no qualifying new vehicle purchase is given, and missed the $2,000 OBBBA charitable deduction for non-itemizers. It also deducted the $144 traditional IRA, which the active-participant phaseout eliminates at this AGI. Together these left taxable income at $125,847 instead of $125,529.30, so tax came out about $70 too high even though the $6,320 of credits was correct."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It treated only $300 per child ($600 total) of the CTC as nonrefundable. In fact the full $2,200 per child offsets tax first, and only any excess is refundable. It also used a pre-OBBBA $30,000 standard deduction, 2025 brackets, and a 20% CDCC instead of 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated about $19,900 of tax before credits and then added a made-up AMT add-back to reach $24,500. AMT does not apply to this standard-deduction return, and nonrefundable credits of $6,320 must lower the $17,031 regular tax, not raise it."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It applied the CDCC at 20% ($1,200) instead of the OBBBA 32% rate ($1,920) for joint AGI just above $150,000. It also deducted $1,575 of auto-loan interest with no qualifying vehicle and left out the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It counted only $600 of the CTC as nonrefundable instead of the full $4,400, and used a 20% CDCC instead of 32%. It also added the $540 short-term gain but ignored the $7,018 long-term loss, so it never took the $3,000 capital-loss deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law, with four personal exemptions and a $750 phased-out CTC. OBBBA made the TCJA structure permanent: no exemptions, a $32,200 standard deduction plus $2,000 charitable deduction, and a $2,200-per-child CTC. It also used a 20% CDCC instead of 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired, taking $21,200 of personal exemptions and a $2,000 CTC. It also subtracted employer-sponsored insurance premiums from wages to reach AGI of about $135k, instead of AGI of $159,729 with the $32,200 + $2,000 deductions and the $4,400 CTC and $1,920 CDCC of permanent OBBBA law."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,700 standard deduction and no $2,000 charitable deduction, a $2,000-per-child CTC ($4,000) instead of $2,200 ($4,400), and a 20% CDCC ($1,200) instead of 32% ($1,920)."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It modeled 2026 as the post-TCJA-expiration year, with $20,200 of personal exemptions, itemized deductions, and a $1,000-per-child CTC. OBBBA made the TCJA rules permanent, so the correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, the $4,400 CTC, and a 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It never subtracted any nonrefundable credits; its $17,822 is about the gross regular tax. It left out the $4,400 CTC and the $1,920 CDCC, and its AGI of $164,481 also skipped the $3,000 capital-loss deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, with personal exemptions, itemizing, and a $700 phased-out CTC. It also lowered AGI to $135,169 by subtracting employer insurance premiums. Under permanent OBBBA law, AGI is $159,729, the $32,200 standard deduction plus $2,000 charitable deduction apply, and credits are a $4,400 CTC and a $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA law, with $20,200 of personal exemptions, itemized deductions, an AGI of about $135k, and a $750 phased-out CTC. It used a 20% CDCC instead of OBBBA's $32,200 + $2,000 deductions, $4,400 CTC, and 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation. Its $11,463 is about $750 above the correct $10,711, which matches taking the CDCC at the old 20% rate ($1,200) instead of the OBBBA 32% rate ($1,920) that applies at joint AGI of $159,729."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,105 is about $394 above the correct figure, which matches leaving out the $2,000 OBBBA charitable deduction for non-itemizers (worth $440 at the 22% bracket) while otherwise applying the $32,200 standard deduction and $6,320 of credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It explicitly used itemized deductions and personal exemptions, which means it applied the expired pre-TCJA regime. Permanent OBBBA law for 2026 has no exemptions and uses the $32,200 standard deduction plus the $2,000 charitable deduction, with a $4,400 CTC and $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $10,817 is $106 above the correct figure, which matches about $480 of excess taxable income at the 22% bracket. That fits leaving out part of the above-the-line or non-itemizer deductions (the $337.50 educator expense and part of the $2,000 charitable deduction) while the $6,320 of credits was right."
+us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It made up a $15,600 CTC instead of $4,400 and a saver's credit of $1,404, which is unavailable at AGI near $160k. It limited the CDCC to one child's $3,000 at 20% instead of $6,000 at 32%, and also ignored the 401(k) exclusion and the capital-loss deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It treated only $800 of the CTC as nonrefundable. The full $4,400 offsets tax first, since liability exceeds it. It also used a 20% CDCC instead of 32% and left out the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $17,278 is roughly the gross regular tax, which means it barely applied the nonrefundable credits. The $4,400 CTC and $1,920 CDCC ($6,320 total) must be subtracted from the $17,031 regular tax."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It gave no numeric derivation. Its $14,696 is about $4,000 above the correct figure, which matches subtracting only a fraction of the $6,320 of nonrefundable credits (the full $4,400 CTC plus $1,920 CDCC) from the $17,031 regular tax."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used the CDCC at 20% ($1,200) instead of the OBBBA 32% rate ($1,920) for joint AGI of about $159.7k. It also took only the $32,200 standard deduction without the $2,000 non-itemizer charitable deduction. Those two errors account for its $1,169 overstatement."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It subtracted a $1,200 CDCC at the old 20% rate instead of the $1,920 the OBBBA rate schedule gives (35% stepped down 1 point per $4,000 of joint AGI over $150,000, which is 32%). That understated nonrefundable credits by $720."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It took the CDCC at 20% ($1,200) instead of 32% ($1,920). Its taxable income of $127,528 also leaves out the $2,000 OBBBA charitable deduction for non-itemizers, which should bring taxable income to $125,529."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC ($4,000) instead of OBBBA's $2,200 ($4,400), and a 20% CDCC instead of 32%. It also reached taxable income of about $128k through itemizing instead of the $32,200 standard deduction plus the $2,000 charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. It also overstated regular tax at about $18,360 on taxable income near $127.6k, where the 2026 MFJ brackets give about $17.5k."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It applied the CDCC at 20% ($1,200) instead of the OBBBA 32% ($1,920). Its taxable income of $127,384 also leaves out the $2,000 charitable deduction for non-itemizing joint filers."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Everything matched except the CDCC rate: it used 30% ($1,800) instead of 32% ($1,920). The OBBBA phase-down above $150,000 cuts the 35% rate by 1 point per $4,000 for joint filers, so $9,729 of excess AGI means 3 steps and a 32% rate, not 5."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a rough $14,000 with no derivation. The correct path is $17,031 of regular tax on $125,529 taxable income (after the $32,200 standard deduction and $2,000 charitable deduction), minus the $4,400 CTC and $1,920 CDCC. Its answer overstates the result by about $3,300, consistent with under-applying those credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the TCJA sunset: restored 2017-style brackets, $21,200 of personal exemptions, and a CTC fully phased out. OBBBA made TCJA permanent, so the $32,200 standard deduction plus $2,000 charitable deduction, current brackets, the $4,400 CTC, and the 32% CDCC apply."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed 2026 reverted to pre-TCJA law, with personal exemptions, a $16,600 standard deduction, 10/15/25% brackets, and a $1,000 CTC phased out above $110k. Under permanent OBBBA law, the $4,400 CTC is fully available and the CDCC is 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It modeled a TCJA expiration, with $5,400 personal exemptions, 10/15/25% brackets, and a fully phased-out CTC. OBBBA made the TCJA structure permanent, with a $32,200 standard deduction plus a $2,000 charitable deduction, a $4,400 CTC, and a 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 10/15/25% brackets, an AGI of $134,987 that wrongly excludes employer insurance premiums, and a $751 phased-out CTC. Permanent OBBBA law gives AGI of $159,729, a $4,400 CTC, and a 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of the OBBBA 32% rate ($1,920) and left out the $2,000 charitable deduction for non-itemizers. Its regular tax of about $17,508 is therefore too high and its credits are too low."
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed large charitable deductions wipe out the liability. In fact itemized deductions fall below the $32,200 standard deduction, and non-itemizers get only a $2,000 charitable deduction. Taxable income of $125,529 produces $17,031 of tax, and $6,320 of credits cannot reduce that to zero."
+us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of 32% ($1,920), deducted the $144 traditional IRA despite the active-participant phaseout, and left out the $2,000 non-itemizer charitable deduction. It also used 2025 bracket boundaries and overstated regular tax at $17,852."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It subtracted the full $6,478 net capital loss instead of capping it at $3,000. It used a $2,000-per-child CTC and a 20% CDCC to reach $11,730, then submitted $24,711.50, which contradicts its own computation and exceeds even gross tax."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It never subtracted the $4,400 CTC and applied only a $1,200 CDCC. It also invented a $4,160 educator deduction, ignored the $3,000 capital-loss deduction, and used a $29,994 standard deduction instead of $32,200 plus the $2,000 charitable deduction."
us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Its own reasoning computed pre-credit federal tax of about $17,200 and stated that the $4,000 CTC reduces it to about $13,200, which under §24(d) leaves zero credit unused by the liability limitation, then abandoned that conclusion and reported $1,200 as an unexplained residual. It applied only the earned-income prong (15% × ($165,000 − $2,500) = $24,375) and never the controlling ceiling that the ACTC cannot exceed the CTC disallowed by tax liability. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200."
us,scenario_099,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the statutory per-child refundable cap ($1,700 × 2 = $3,400) as an automatic refund for any filer with earnings above the $2,500 floor, skipping §24(d)'s requirement that the ACTC equals only the CTC remaining after the credit is applied against tax liability. With roughly $17,400 of federal income tax before credits, the full $4,400 of 2026 CTC is absorbed nonrefundably, so the refundable amount is $0. It also used the 2025 $1,700 cap instead of the 2026 indexed figure."
us,scenario_099,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly identified the OBBBA 2026 $2,200 per-child CTC and the $1,800 per-child refundable cap, then treated that cap as a guaranteed refund because earnings exceed the $2,500 threshold and AGI is under the $400,000 phase-out. The refundable portion is the lesser of the earned-income formula, the per-child cap, and the CTC left over after offsetting tax; on about $127,000 of taxable income the roughly $17,400 tax consumes all $4,400 of CTC nonrefundably, leaving $0."
us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the model performed no substantive computation to evaluate. The correct derivation gives $0: about $17,400 of pre-credit federal tax fully absorbs the $4,400 CTC under the §24(d) liability limitation, and $159,676 of income is far past the two-child MFJ EITC phase-out."
us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so no substantive reasoning exists to fault. The correct derivation gives $0: the roughly $17,400 of federal tax before credits leaves no CTC unused for the ACTC, and EITC is zero at $159,676 of income."
us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It invented a ""$300 Other Dependent Credit portion"" for two children aged 9 and 4 who are qualifying children; the §24(h)(4) other-dependent credit is $500, applies only to dependents who are not qualifying children, and is wholly nonrefundable, so it can never enter federal_refundable_credits. Its own explanation states that pre-credit tax absorbs the nonrefundable credits and no ACTC remains — a derivation that yields $0 — yet it submitted $600."
-us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Compared about $166,694 of gross income to the 130% FPL federal free-meal limit and stopped there. It never applied California's universal school meals program, which gives free meals to the 9-year-old K-12 child at any income."
-us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Treated the 130% FPL income test and categorical eligibility as the only ways to qualify for free meals. It missed California's universal free school meals program, which covers every enrolled K-12 student regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL income test and also cited asset holdings, although the school meal programs have no asset test. It overlooked California's universal school meals program, under which the 9-year-old K-12 child receives free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Used the 185% FPL threshold as the free-meal limit, but 185% is actually the reduced-price cutoff. More importantly, it ignored California's universal school meals program, which makes the K-12 child eligible for free meals at any income."
-us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Relied only on the federal 130% FPL free-meal income test. It never considered that California provides universal free school meals to every enrolled K-12 student, so the 9-year-old qualifies despite $165,000 in wages."
-us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Considered only the 130% FPL income test and categorical eligibility. It omitted California's universal free school meals program, which gives the K-12 child free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Denied eligibility on the 130% FPL income test alone. It missed California's universal school meals program, which overrides the income test and makes the 9-year-old K-12 child eligible for free meals."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the NSLP rule of income at or below 130% FPL (about $42,900) as the sole test. It overlooked California's universal school meals program, which gives free meals to all enrolled K-12 students at any income."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Compared $165,000 in wages to the federal 130%/185% free and reduced-price thresholds and never applied California's universal school meals program. That program gives the 9-year-old K-12 child free meals regardless of household income."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,Used the 130% FPL federal free-meal threshold as decisive and ignored California's universal free school meals program. Under that program the K-12 child is eligible at any income.
-us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Applied only the 130% FPL free-meal limit. It missed California's universal school meals program, which makes the enrolled K-12 child eligible for free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Denied eligibility because income exceeds 130% FPL and did not account for California's universal free school meals program. That program covers every K-12 student, including the 9-year-old, at any income."
-us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Treated income above 130% FPL as disqualifying. It never applied California's universal school meals program, which supersedes the federal income test and gives the K-12 child free meals."
-us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,Relied on the federal 130% FPL free-meal cutoff alone and missed California's universal free school meals program. Under that program the 9-year-old K-12 child qualifies regardless of income.
-us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Denied eligibility on an income threshold. It ignored that California's universal school meals program has no income threshold and gives free meals to every enrolled K-12 student, including Child 1."
-us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Applied the 130% FPL federal free-meal test and never applied California's universal school meals program. That program makes the 9-year-old K-12 child eligible for free meals at any income.
-us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Used an income threshold to deny free meals. It missed California's universal free school meals program, which drops the income test entirely for enrolled K-12 students."
-us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,Denied eligibility on the federal income threshold and overlooked California's universal school meals program. That program provides free meals to the K-12 child regardless of household income.
-us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"Treated the 130% FPL federal free-meal limit as decisive. It never considered California's universal free school meals program, which covers every enrolled K-12 student at any income."
-us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Denied free meals on an income threshold. It missed California's universal school meals program, under which the 9-year-old K-12 child qualifies for free meals with no income test."
-us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Compared about $156,588 of AGI to 130% FPL and denied eligibility. It never applied California's universal school meals program, which makes the K-12 child eligible for free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Applied the 130% FPL limit (about $43,000) as the only route to free meals and missed California's universal free school meals program. That program covers the 9-year-old K-12 student at any income."
-us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"Denied eligibility because income exceeds the free-meal thresholds. It ignored California's universal school meals program, which has no income threshold for enrolled K-12 students."
-us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed wage income above federal thresholds makes the household ineligible. It never applied California's universal free school meals program, under which PolicyEngine gives the K-12 child the FREE tier regardless of income."
-us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL free-meal limit and checked for categorical eligibility, but missed California's universal school meals program. That program makes the K-12 child eligible for free meals at any income."
-us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Denied free meals on an income threshold and overlooked California's universal school meals program. That program gives every enrolled K-12 student, including Child 1, free meals with no income test."
-us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Relied on the federal free-meal income threshold. It missed California's universal free school meals program, which makes the 9-year-old K-12 child eligible regardless of income."
-us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied only the federal free-meal income threshold and ignored California's universal school meals program. Under that program PolicyEngine gives the K-12 child positive free-meal support at any income.
-us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,Treated high income as disqualifying. It missed that California's universal school meals program provides free meals to all enrolled K-12 students regardless of income.
-us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Considered only the 130% FPL test and categorical eligibility. It never applied California's universal free school meals program, which gives the K-12 child free meals at any income."
-us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked the 130% FPL limit and categorical certification through SNAP, TANF, or Medicaid, but overlooked California's universal school meals program. That program is a separate route that makes the 9-year-old K-12 child eligible regardless of income."
-us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Applied the 130% FPL (about $43,000) federal free-meal limit as decisive. It missed California's universal free school meals program, which covers the enrolled K-12 child at any income."
-us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Tested income against 130% of the 2026 poverty guideline and looked for categorical factors such as SNAP, TANF, or foster care. It omitted California's universal school meals program, which gives free meals to the K-12 child regardless of income."
-us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no parseable value or explanation for free_school_meals_eligible, so there was no answer to score. The correct derivation yields 1, because California's universal school meals program makes the 9-year-old K-12 child eligible for free meals."
-us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"Denied eligibility on free-meal income limits. It never applied California's universal school meals program, which has no income limit for enrolled K-12 students."
-us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Treated income at or below 130% FPL (about $43,000) as a requirement for free meals. It missed California's universal free school meals program, under which the K-12 child qualifies at any income."
-us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"Named California but still applied only the federal 130% FPL test. It missed California's universal school meals program, which gives every enrolled K-12 student free meals regardless of household income."
-us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,Denied eligibility on a free-meal income threshold and overlooked California's universal school meals program. That program makes the 9-year-old K-12 child eligible for free meals at any income.
+us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Compared about $166,694 of income with the federal 130% FPL free-meal cutoff and stopped there. It never applied California's universal free school meals program, which gives the 9-year-old K-12 student free meals at any income."
+us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,Ruled the household out because income is above 130% FPL and it has no categorical eligibility. It missed that California's universal school meals program makes free meals available to every enrolled K-12 student with no income or categorical test.
+us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Used the federal 130% FPL income test and added an asset limit that school meals do not have. It missed California's universal free school meals program, under which the 9-year-old K-12 child gets free meals regardless of income or assets."
+us,scenario_099,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,Said income is far above the free-meal limit. It ignored that California's universal school meals program has no income limit and gives free meals to the enrolled K-12 child.
+us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Tested income against an FPL threshold and gave it wrongly as 185%, which is the reduced-price cutoff; the free-meal cutoff is 130%. More importantly, it missed California's universal free school meals program, which makes the K-12 child eligible regardless of income."
+us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Treated the federal 130% FPL income test as the only route to free meals. California's universal meals program gives free meals to all enrolled K-12 students, which yields positive support for the 9-year-old."
+us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,Concluded No based on income above 130% FPL and no categorical eligibility. It missed that California's universal free school meals program sets the meal tier to FREE for the K-12 child without regard to income.
+us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only the federal 130% FPL free-meal income test. It never considered California's universal school meals program, which gives the 9-year-old free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Compared income with roughly $42,900, its figure for 130% FPL, under the federal NSLP rules. It missed that California's universal meals program overrides the income test and gives free meals to every K-12 student."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Used only the federal 130%/185% FPL thresholds and gave their dollar amounts in reverse order. It never applied California's universal free school meals program, under which the enrolled 9-year-old gets free meals regardless of the household's $165,000 income."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Ruled the household ineligible because about $165k is above 130% FPL. It missed California's universal school meals program, which provides free meals to the K-12 child with no income limit."
+us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Applied the federal 130% FPL free-meal limit. It missed that California's universal free school meals program makes every enrolled K-12 student eligible for free meals regardless of income.
+us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Answered No only because income is above 130% FPL. It ignored California's universal school meals program, which gives the 9-year-old free meals at any income."
+us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Treated income above 130% FPL as disqualifying. California's universal meals program sets the tier to FREE for all enrolled K-12 students, so the household receives positive free-meal support."
+us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied only the federal 130% FPL test. It missed California's universal free school meals program, which covers the K-12 child regardless of household income."
+us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Assumed an income threshold decides free-meal eligibility. In California, the universal school meals program gives free meals to every enrolled K-12 student with no income test."
+us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Used the federal 130% FPL cutoff to rule the children ineligible. It missed that California's universal meals program makes the 9-year-old K-12 student eligible for free meals regardless of income.
+us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,Applied an income threshold that does not bind in California. The state's universal school meals program gives the enrolled K-12 child free meals at any income.
+us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"Ruled the household out on income. It never applied California's universal free school meals program, which removes the income test for all K-12 students."
+us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"Applied the federal 130% FPL free-meal guideline. It missed California's universal meals program, under which the 9-year-old receives free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Used an income threshold to deny free meals. California's universal school meals program gives free meals to every enrolled K-12 student, which produces positive support for this household."
+us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Compared about $156,588 of AGI with 130% FPL, using AGI where the program counts gross income. More importantly, it missed California's universal free school meals program, which makes the K-12 child eligible regardless of income."
+us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Compared about $160,000 of income with a 130% FPL limit of about $43,000 under the federal rules. It ignored California's universal school meals program, which has no income test."
+us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,Applied federal free-meal income thresholds. It missed that California's universal school meals program gives free meals to the enrolled 9-year-old at any income.
+us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed wage income above the thresholds rules out free meals. It missed California's universal free school meals program, which makes every K-12 student eligible without an income test."
+us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Relied on the federal 130% FPL limit and the absence of categorical eligibility. It never applied California's universal school meals program, which sets the meal tier to FREE for the K-12 child regardless of income."
+us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Denied eligibility on the free-meal income threshold. California's universal meals program gives free meals to all enrolled K-12 students, so PolicyEngine returns positive support."
+us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied the federal income threshold. It missed California's universal free school meals program, which covers the 9-year-old K-12 student regardless of household income."
+us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Used the federal free-meal income threshold. It missed that California's universal school meals program overrides it and gives free meals to every enrolled K-12 student.
+us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,Treated high income as disqualifying. California's universal school meals program has no income limit and gives free meals to the enrolled K-12 child.
+us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Concluded the free-meal value is zero from the 130% FPL test and the lack of categorical eligibility. It missed California's universal meals program, which gives the K-12 child $1,130.96 a year in free-meal support regardless of income."
+us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked only the federal 130% FPL test and categorical certification through SNAP, TANF or Medicaid. It never considered California's universal free school meals program, which needs neither."
+us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Compared income with a 130% FPL limit of about $43,000. It ignored that California's universal school meals program gives free meals to every enrolled K-12 student at any income."
+us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Applied the federal 130% FPL test and looked for categorical routes such as SNAP, TANF or foster care. It missed California's universal free school meals program, which makes the 9-year-old eligible regardless of income."
+us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"Gave no value or explanation for free_school_meals_eligible, so there was nothing to score. The correct answer is 1, because California's universal school meals program gives free meals to the enrolled K-12 child."
+us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,Applied free-meal income limits. It missed that California's universal school meals program has no income limit and covers the K-12 child.
+us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Compared about $160k of income with a 130% FPL limit of about $43k. It never applied California's universal free school meals program, under which PolicyEngine sets the tier to FREE for the enrolled 9-year-old."
+us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,Explicitly applied the 130% FPL threshold to a California household. It missed that California's universal school meals program gives free meals to all K-12 students regardless of income.
+us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"Denied free meals on an income threshold. California's universal meals program gives free meals to every enrolled K-12 student, which produces positive support for this household."
us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). Its own sum came to $14,602.50, but it submitted $14,118, which no step of its derivation supports."
-us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($10,230 SS + $2,392.50 Medicare) and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145. The prompt explicitly includes that tax in payroll_tax."
-us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed California has no state-mandated employee payroll tax, so it dropped CA SDI. That tax is 1.3% of all wages in 2026 ($2,145), and leaving it out left only FICA of $12,622.50."
-us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a CA SDI rate of 1.2% ($1,980) instead of 2026's 1.3% ($2,145). Its own recomputed total was $14,602.50, yet it submitted $13,197.50, an arithmetic inconsistency."
-us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It computed CA SDI at 1.1% ($1,815) and then dropped it on the assumption that the benchmark 'focuses on federal FICA'. The prompt defines payroll_tax to include mandatory employee state payroll taxes, and the correct SDI is 1.3% × $165,000 = $2,145."
-us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It reported only SS ($10,230) and Medicare, truncating Medicare to $2,392. It left out California's mandatory 1.3% employee SDI ($2,145) entirely."
-us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly treated CA SDI as uncapped but applied the 2024 rate of 1.1% ($1,815). The 2026 rate is 1.3% ($2,145), so its answer is $330 short."
-us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It first called CA SDI voluntary, then briefly added it at 1.1%, and finally submitted $12,903.50, a figure none of its steps produce. The correct answer adds mandatory SDI at 1.3% of $165,000 ($2,145) to FICA of $12,622.50."
-us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145) and rounded the result to $14,438."
-us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the employer-sponsored insurance premiums ($24,597) from gross wages as pre-tax deductions, cutting the FICA/SDI base to $140,403. These are listed employer premiums, not employee salary reductions, so the base is the full $165,000. It also used a 1.1% SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $24,597 of employer-sponsored insurance premiums as pre-tax salary reductions and taxed only $140,403 of wages. SS, Medicare and SDI all apply to the full $165,000 of gross wages. It also used a 1.1% SDI rate instead of 2026's 1.3%."
-us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145), which leaves it $165 short."
-us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a CA SDI rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% on $165,000 ($2,145)."
-us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare. It left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145, which the payroll_tax definition includes."
-us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted $24,597 of employer-sponsored insurance premiums from gross wages as if they were pre-tax employee deductions, taxing only $140,403. The FICA and SDI base is the full $165,000. It also used a 1.1% SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 uncapped rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"By its own description it included only Social Security and Medicare, leaving out California's mandatory 1.3% SDI ($2,145). Its $12,963 does not even match correct FICA of $12,622.50 on $165,000 of wages."
-us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It said it included CA SDI, but full-wage FICA of $12,622.50 leaves only $657.50 of SDI in its $13,280 total. Correct SDI is 1.3% × $165,000 = $2,145, so it badly understated the state disability tax."
-us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 equals exactly 8.85% × $140,403. That means it took $24,597 of employer-sponsored insurance premiums out of the wage base and used a 1.2% SDI rate. The correct result is 7.65% FICA plus 1.3% SDI on the full $165,000."
-us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $14,103 leaves about $1,480 for SDI above full-wage FICA of $12,622.50, roughly a 0.9% rate. The 2026 CA SDI rate is 1.3%, which gives $2,145."
-us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It applied 6.2% to the wage base cap ($176,100) instead of to actual wages, then landed on an unexplained $12,632.40. Both earners are under the cap, so SS is $10,230. It also left out California's mandatory 1.3% SDI ($2,145) entirely."
-us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"It correctly treated CA SDI as mandatory and uncapped but used a 1.2% rate ($1,980) instead of 2026's 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It treated both adults as subject to employee FICA only and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145. It also truncated the FICA total to $12,622."
-us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It counted only employee FICA and left out California's mandatory 1.3% SDI ($2,145). Its $13,577 also overstates FICA, which is exactly $12,622.50 (7.65% of $165,000)."
-us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"It estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It stated a CA SDI rate of about 1.2% instead of the 2026 rate of 1.3%. Its $14,312.40 does not even match its own inputs, since FICA of $12,622.50 plus 1.2% SDI ($1,980) is $14,602.50."
-us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"It estimated CA SDI at $1,980 (1.2% of wages) instead of the 2026 rate of 1.3%, which gives $2,145."
-us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used CA SDI of about $1,980 (a 1.2% rate) instead of 1.3% × $165,000 = $2,145, then rounded to $14,603."
-us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It computed only employee SS and Medicare (about $12,623) and left out California's mandatory 1.3% employee SDI of $2,145."
-us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It correctly added mandatory CA SDI but used a 1.2% rate ($1,980) instead of 2026's 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"It treated CA SDI as uncapped but applied 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $24,597 of employer-sponsored insurance premiums from gross wages and computed FICA on only $140,403 instead of the full $165,000. It also left out California's mandatory 1.3% SDI ($2,145) entirely."
-us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of 2026's 1.3% ($2,145) and rounded the total to $14,603."
-us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The correct answer is FICA of $12,622.50 plus CA SDI of $2,145, for $14,767.50."
-us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The correct answer is FICA of $12,622.50 plus CA SDI of $2,145, for $14,767.50."
-us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Its per-person employee FICA ($6,502.50 + $6,120 = $12,622.50) was right, but it reported $25,290, roughly double. That number is consistent with adding the employer share, which the prompt excludes. It also left out California's mandatory 1.3% SDI ($2,145)."
-us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that no employee-side state payroll tax applies and stopped at FICA of $12,622.50. California's SDI is a mandatory employee payroll tax of 1.3% on all wages in 2026 ($2,145)."
-us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It recognized that the prompt's definition includes mandatory CA SDI and computed it, though at an outdated 1.1% rate with a wage cap that no longer exists. It then went back and submitted FICA only ($12,622.50), leaving out the 1.3% SDI of $2,145."
-us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied CA SDI to the head's wages only, ignoring the spouse's $80,000. It also got the arithmetic wrong: 1.2% of $85,000 is $1,020, not $472.20. Correct SDI is 1.3% of both earners' $165,000, which is $2,145."
+us,scenario_099,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It applied CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $14,118, which does not match its own $14,602.50 total."
+us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($12,622.50) and left out California's mandatory employee SDI. SDI is 1.3% of $165,000 in 2026, or $2,145."
+us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It stated that California has no mandatory employee payroll tax. That dropped the 1.3% uncapped SDI contribution ($2,145), so it reported FICA alone at $12,622.50."
+us,scenario_099,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It said California has no mandatory state payroll tax, which left out the $2,145 SDI contribution (1.3% of wages). It also submitted $12,242, which does not match its own $10,230 + $2,393 FICA sum."
+us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It used the 1.2% SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $13,197.50, which does not match the $14,602.50 total it computed."
+us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It computed CA SDI at an outdated 1.1% rate and then left it out of the total, treating payroll_tax as federal FICA only. The prompt explicitly includes mandatory state employee payroll taxes, so the 1.3% SDI ($2,145) belongs in the total."
+us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It reported only Social Security and Medicare (rounded to $12,622) and left out the mandatory CA SDI contribution of 1.3% × $165,000 = $2,145."
+us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It correctly included uncapped CA SDI but used the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145), so its total was $330 short."
+us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wavered on whether to include SDI, tried it at 1.1%, and then submitted $12,903.50, a figure none of its own steps produce. The correct total adds 1.3% SDI ($2,145) to FICA of $12,622.50."
+us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $24,597 in employer-sponsored insurance premiums from the wage base, but the reference applies FICA and SDI to the full $165,000 in gross wages. It also used a 1.1% SDI rate instead of 1.3%."
+us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It cut the payroll tax base to $140,403 by removing the employer-sponsored insurance premiums, when the full $165,000 in gross wages is the base. It also applied SDI at 1.1% instead of the 2026 rate of 1.3%."
+us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied CA SDI at the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed only federal Social Security and Medicare and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145."
+us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It took the $24,597 in employer-sponsored insurance premiums out of FICA/SDI wages, but the payroll taxes are computed on the full $165,000 in gross wages. It also used 1.1% for SDI instead of the 2026 rate of 1.3%."
+us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It used the outdated 1.1% CA SDI rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out the mandatory CA SDI contribution of 1.3% of wages ($2,145). Its $12,963 also does not equal 7.65% × $165,000 = $12,622.50."
+us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It said it included CA SDI, but $13,280 leaves only about $657 for SDI on top of the $12,622.50 in FICA. The 2026 SDI rate is 1.3% of all $165,000 in wages, or $2,145."
+us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied payroll taxes to a reduced 'taxable wages' base instead of the full $165,000 in gross wages, and it understated SDI. FICA alone on gross wages is $12,622.50, before adding $2,145 of SDI at 1.3%."
+us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $14,103 implies only about $1,480 of CA SDI (roughly 0.9%) on top of $12,622.50 in FICA. The 2026 SDI rate is 1.3% with no wage cap, which gives $2,145."
+us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It left out California SDI entirely, and it produced Social Security figures ($10,918.20, $10,465.20) that are not 6.2% of $165,000. The correct total is $10,230 + $2,392.50 + 1.3% SDI ($2,145)."
+us,scenario_099,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It correctly treated CA SDI as mandatory and uncapped but used the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It treated the household's wages as subject to employee FICA only and left out California's mandatory 1.3% employee SDI ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It counted only FICA and left out the 1.3% CA SDI ($2,145). Its $13,577 also does not equal 7.65% × $165,000 = $12,622.50."
+us,scenario_099,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It cited an approximate 1.2% SDI rate, but its $14,312.40 total implies only about $1,690 of SDI. The 2026 rate is 1.3% of $165,000, or $2,145."
+us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It estimated CA SDI at about $1,980 (1.2%) instead of the 2026 amount of $2,145 at 1.3% of uncapped wages."
+us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare (about $12,623) and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145."
+us,scenario_099,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It correctly treated CA SDI as uncapped but applied 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It reduced FICA wages to $140,403 by subtracting the employer-sponsored insurance premiums, when the full $165,000 in gross wages is the base. It also left out California's mandatory 1.3% SDI ($2,145)."
+us,scenario_099,payroll_tax,inkling,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no parseable answer to score."
+us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no parseable answer to score."
+us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Its per-person FICA amounts ($6,502.50 + $6,120) add up to $12,622.50, but it submitted $25,290, roughly double that. It also left out the 1.3% CA SDI ($2,145)."
+us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly concluded that no employee-side state payroll tax applies in California. That left out the mandatory 1.3% SDI on all wages ($2,145)."
+us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It recognized that SDI counts as a mandatory state payroll tax but used an outdated 1.1% rate with a wage cap. It then left SDI out of the final $12,622.50 entirely, when uncapped 1.3% SDI ($2,145) belongs in the total."
+us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied SDI only to the head's wages and miscomputed even that ($472.20 is not 1.2% of $85,000). The 2026 SDI is 1.3% on both earners' combined $165,000, or $2,145."
us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -7868,52 +8064,53 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_
us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It got taxable income roughly right (~$138,791 after $21,275 itemized) but priced it at ≈$6,940, about $1,160 more than the frozen reference's $5,778 and about $1,070 more than the $5,871 that the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, give on $138,791. Both that schedule and the engine's projected one keep ~$139k in the 8% bracket. It then reported $6,222, which contradicts its own post-credit figure of $5,684; its charity deduction without the 0.5%-of-AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a ~$4,406 medical deduction by counting the $2,500 premium twice per adult (both premium fields) and adding over-the-counter items. The real deductible medical costs ($5,000 of premiums plus $5,000 of other medical) are below the 7.5%-of-AGI floor of ~$11,969, so the medical deduction is zero. Itemized deductions are $20,477 (charity after the 0.5% AGI floor plus property tax), not $25,681, which drove taxable income down to ~$134,241."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used a $4,754 standard deduction instead of itemizing $20,477 of charity (after the 0.5% AGI floor) and property tax, which exceeds the $11,412 CA MFJ standard deduction. It also deducted unreimbursed employee expenses above the line. In place of the $1,284.46 personal/dependent exemption credits it subtracted an invented $146 'child tax credit' and a $543 educator-expense 'credit'."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income (~$138,309) was close to the frozen reference's $139,108, but it guessed bracket tax at ~$6,200, above both the $5,778 the frozen reference's projected 2026 CA MFJ schedule produces and the $5,897 that the 2025 schedule, which the release's California convention (c_ca_hold_2025) holds for 2026, gives on $139,108. It then raised its own ~$4,680 net to $5,959 by 'adjusting for SDI'. SDI is a payroll tax and is not part of state income tax."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It started from $165,000 of gross wages without excluding the $3,088 of traditional 401(k) deferrals and added back the educator deduction, which inflated CA AGI to ~$163,010 instead of $159,585. It then priced $141,735 as if it were deep in the 9.3% bracket (~$7,973). Under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected 2026 MFJ schedule, income near $139k sits in the 8% bracket (for $5,778 of tax in the frozen reference)."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of ~$138,000 was about right, but it priced it at ~$7,700, roughly $1,900 above the frozen reference's projected 2026 CA MFJ schedule ($5,778 at $139,108) and $1,800 above the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,897 there). It then subtracted only ~$450 of exemption credits, although its own listed components add up to ~$1,300. The personal-plus-dependent credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection)."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity instead of $11,823 after the 0.5%-of-AGI charitable floor, and it left the $338 educator deduction out of CA AGI. Taxable income came to $138,647 instead of $139,108, so its bracket tax of $5,729 fell short of $5,778 and the final answer landed $53 low."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It valued each adult's personal exemption credit at ~$450 (the dependent amount) instead of about $150, which put total credits at $1,800 instead of $1,284.46. It also priced $138,791 at $5,989 using guessed brackets. Its taxable income omitted the 0.5% AGI charitable floor and the IRA/educator adjustments, and the overstated credits account for most of the $305 shortfall."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It listed the personal and dependent exemption credits but never subtracted them: its reported $6,800 equals its own pre-credit tax, while the credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference). It also guessed taxable income at ~$150,000 instead of $139,108 by under-counting itemized deductions, and it applied a federal-style $10k SALT cap that California does not have."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a medical deduction of about $4.4k, counting the $2,500 premium twice per adult plus over-the-counter items. The household's actual deductible medical costs ($10,000) are below the 7.5%-of-AGI floor, so medical adds nothing. Itemized deductions are $20,477, not ~$25.7k, and taxable income is $139,108, not ~$134k."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It left the $3,088 of 401(k) deferrals in CA AGI ($163,306 instead of $159,585). It then added a $2,752 medical deduction built on double-counted premiums, although deductible medical costs are below the 7.5% floor. It overpriced the bracket tax at ~$6,042 and subtracted only ~$1,000 of dependent credits, leaving out the two personal exemption credits (the total is $1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026, and $1,284.46 in the frozen reference)."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $24,597 of employer-sponsored insurance premiums from income, cutting CA AGI to ~$134,987; the listed wages are already the taxable gross, so CA AGI is $159,585. The lower AGI shrank the 7.5% medical floor and produced a $1,276 medical deduction that should not exist. Taxable income came out at $112,436 instead of $139,108."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $24,597 of employer-sponsored insurance premiums from AGI (~$135,069 instead of $159,585). Against that reduced floor it claimed a ~$6,270 medical deduction by counting the $2,500 premium twice per adult plus over-the-counter items. With no medical deduction, itemized deductions are $20,477 and taxable income is $139,108, not $107,524."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $4,395 medical deduction by measuring $16,400 of spending (the $2,500 premium counted twice per adult, plus over-the-counter items) against a $12,005 floor. The actual $10,000 of premiums and medical costs is below 7.5% of AGI, so itemized deductions are only charity after the 0.5% AGI floor plus property tax ($20,477). That makes taxable income $139,108, not $134,396."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted about $1,180 of personal and dependent exemption credits instead of $1,284.46. Its answer implies ~$5,845 of pre-credit tax, above the $5,778 the frozen reference's projected 2026 CA MFJ brackets produce on $139,108 of taxable income but in line with the $5,844 behind the exclusion's corrected value ($4,588.48 plus the $1,256 of 2025 exemption credits that the release's California convention, c_ca_hold_2025, holds for 2026); against that value its $76.52 overstatement is the credit shortfall."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $7,845 matches CA bracket tax on roughly $163k of income, meaning it taxed near-gross income with no itemized deduction and no exemption credits. The frozen reference subtracts $20,477 of itemized deductions to reach $139,108 of taxable income, then $5,778 of tax on the engine's projected 2026 schedule less $1,284.46 of exemption credits; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $24,597 of employer-sponsored insurance premiums from CA AGI ($135,169 instead of $159,585). It then claimed a ~$4,860 medical deduction that the 7.5%-of-AGI floor eliminates, and it subtracted only ~$600 of exemption credits instead of $1,284.46."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no numbers. $2,570 plus the $1,284 exemption credits implies ~$3,850 of pre-credit tax, which corresponds to taxable income near $108k. That is about $31k below the correct $139,108, the same shortfall that comes from excluding the $24,597 of employer-sponsored insurance premiums from income and claiming a medical deduction the 7.5% floor eliminates."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $8,725 is more than the CA bracket tax on the household's entire ~$165k of gross wages, so it applied rates with no itemized deduction and no exemption credits. The frozen reference computes $139,108 taxable, $5,778 of tax, less $1,284.46 of credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,128 is $366 low, which matches adding a ~$4.5k medical deduction (taxable income ~$134.5k). The household's deductible medical costs are below the 7.5%-of-AGI floor, so itemized deductions stop at $20,477 and taxable income is $139,108."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,056 is $438 low, which equals 8% of about $5.5k in excess deductions, the pattern produced by claiming a medical deduction the 7.5%-of-AGI floor eliminates. The only allowable itemized deductions are charity after the 0.5% AGI floor ($11,823) and property tax ($8,654)."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $5,245 matches taking the $11,412 CA standard deduction (taxable ~$148,173, tax ~$6,500, less $1,284 of credits ≈ $5,220) instead of itemizing $20,477 of charity and property taxes."
-us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the charitable gifts did not exceed its $6,566 standard deduction. In fact charity after the 0.5% AGI floor ($11,823) plus $8,654 of property tax totals $20,477, well above the $11,412 CA MFJ standard deduction, so the household itemizes. It also subtracted the $338 educator expense as if it were a credit, left out the $1,284.46 exemption credits, and reported $3,202.96, which matches none of its own intermediate figures."
-us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the ~$11,500 standard deduction instead of itemizing $20,477 (charity after the 0.5% AGI floor plus $8,654 property tax), which overstated taxable income by ~$9k. It used ~$620 of exemption credits instead of $1,284.46. It also claimed a ~$240 CA child and dependent care credit, which is unavailable above $100,000 of AGI."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $6,713 matches bracket tax on ~$150k of income with no exemption credits, which means it used the standard deduction instead of $20,477 of itemized deductions and never subtracted the $1,284.46 personal and dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,931 is $437 high, close to the result of pricing ~$139k of taxable income with 2024 CA brackets that were not indexed. Those brackets start 9.3% at $136,700, whereas under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected schedule, this income stays in the 8% bracket (the frozen reference gives $5,778 of tax before its $1,284.46 credits)."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted health insurance premiums from income (CA AGI $134,323 instead of $159,585). It also treated personal and dependent exemptions as deductions from taxable income rather than as the $1,284.46 of credits. It then reported $6,882, about double the ~$3,300 of CA tax its own $101,811 of taxable income produces."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the CA standard deduction ($11,412) instead of itemizing. Charity after the 0.5% AGI floor ($11,823) plus property tax ($8,654) totals $20,477, so its taxable income was about $9k too high."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity and left the $144 IRA and $338 educator deductions out of CA AGI ($160,066), putting taxable income at $138,791; that is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity), where the frozen reference applies the $798 floor and both deductions ($159,585 of CA AGI, $139,108 taxable). Its bracket and credit estimates then understated net tax by ~$86 against the frozen reference's projected 2026 schedule ($5,778 of tax, $1,284.46 of credits) and by ~$233 against the 2025 schedule and credits that the release's California convention (c_ca_hold_2025) holds for 2026 ($4,615.38 on $138,791), which is its whole gap to the fully law-correct $4,615.48 that also adds back the educator expense."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,250 is $244 below $5,778 − $1,284.46, which requires about $3k more in deductions than the correct $20,477 itemized total. That excess comes from deducting charity without the 0.5%-of-AGI floor and claiming medical costs the 7.5%-of-AGI floor eliminates."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It priced taxable income with the 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and used the 2025 exemption credits ($1,256); the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. It also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It itemized charity and property tax, which leaves ~$5,800 of bracket tax, but its $5,138 answer implies only about $660 of exemption credits. The two personal and two dependent exemption credits total $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection). It also deducted full charity without the 0.5%-of-AGI floor."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted full charity without the 0.5% AGI floor and dropped the educator deduction (taxable income $138,647 instead of $139,108). It then priced that income with the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, giving ~$5,860 of pre-credit tax where the frozen reference's projected 2026 schedule gives $5,778 on $139,108. That schedule difference drives its $72 overstatement against the frozen reference; against the exclusion's corrected value, $4,588.48, it is $22.48 low, mainly because its answer then implies about $1,294 of exemption credits where the convention's 2025 amounts give $1,256."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used the 'latest published' 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and the 2025 exemption credits of $1,256; the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. To reach that taxable income it also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round ~$4,500 guess based on '~165k income' without working out itemized deductions, brackets or exemption credits. The frozen reference computes $139,108.47 taxable, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income ($138,453) is $655 low because it deducted full charity without the 0.5%-of-AGI floor. It then priced that income at ~$5,890, about $160 more than the frozen reference's projected 2026 CA MFJ brackets produce (~$5,726) and about $46 more than the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives (~$5,844), for a net overstatement of $106 against the frozen reference."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the 0.5%-of-AGI floor and left out the IRA and educator adjustments, understating taxable income by $317 ($138,791). It used ~$1,246 of exemption credits instead of $1,284.46. The errors mostly offset and left it $3.26 high."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity instead of $11,823 after the 0.5%-of-AGI floor and left out the educator deduction. That understated taxable income by $617 ($138,491 instead of $139,108), and the answer landed $67 low."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $110,811 is about $28k too low, the shortfall produced by excluding the $24,597 of employer-sponsored insurance premiums from income. It also stated that no nonrefundable credits apply, but CA's personal and dependent exemption credits ($1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026; $1,284.46 in the frozen reference) are not reduced at ~$160k of AGI."
-us,scenario_099,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only $576 of exemption credits instead of $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 (two personal credits of $153 and two dependent credits of $475; the frozen reference's projection gives $1,284.46). It also priced its $138,491 of taxable income at $5,866, above what either that 2025 schedule (~$5,847) or the frozen reference's projected schedule gives; its charity deduction without the 0.5% AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor."
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits. The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48."
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits. The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48."
-us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 without any computation, as if the household owed no CA income tax. In fact $139,108 of taxable income generates $5,897 of bracket tax under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,778 on the frozen reference's projected schedule), and the $1,256 of exemption credits ($1,284.46 in the frozen reference) offsets only part of it."
-us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It asserted that California requires the standard deduction when the federal standard deduction is claimed. California lets filers itemize regardless of the federal election, and $20,477 of itemized charity (after the 0.5% AGI floor) and property tax beats the $11,412 standard deduction. Its taxable income (~$147,800) was therefore about $8.7k too high."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It never excluded the $3,088 of 401(k) deferrals or the $144 IRA deduction, and it netted the full $6,478 capital loss instead of limiting it to $3,000. It then priced $138,063 at only $4,301.50, although the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,813 (the frozen reference's projected 2026 brackets, about $5,700). It also left out the $1,284.46 of personal and dependent exemption credits, claiming none apply."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It invented a $34,000 standard deduction instead of itemizing $20,477, and it priced $130,894 at $8,100 when the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,240 at that income. It left out the $1,284.46 of exemption credits and subtracted a $41 renter's credit that the household's ~$160k AGI makes unavailable."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $21,275 with no 0.5%-of-AGI charitable floor, then overstated bracket tax on about $138,791 as about $6,940, when the 2026 MFJ brackets give about $5,750. Its own figure after $1,256 of exemption credits was $5,684, yet it reported $6,222, adding about $540 with no basis."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a medical deduction of about $4,406, but the two $2,500 premium fields describe the same premium. Correctly counted, out-of-pocket medical is at most $11,400, which is under the 7.5% floor of about $11,969, so the medical deduction is zero. That error cut taxable income to about $134,241 instead of $139,108, and it also skipped the 0.5%-of-AGI floor on charitable gifts."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used an outdated $4,754 CA standard deduction instead of itemizing $20,477 (real estate taxes plus charity, which beat the $11,412 standard deduction for 2026). That gave $155,605 of taxable income instead of $139,108. It then replaced the $1,284.46 personal and dependent exemption credits with a made-up $146 child credit and a made-up $543 educator-expense credit."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction and worked from about $150,000 of taxable income instead of itemizing the $20,477 of real estate taxes and charity, which gives $139,108. Its rough $4,200 does not follow from any bracket calculation; the 2026 MFJ brackets give $5,778.21 on $139,108, less $1,284.46 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached about $4,680 after exemption credits, then it raised the answer to $5,959 by 'adjusting for SDI'. CA SDI is an employee payroll tax and is excluded from state income tax. It also deducted the full $21,275 with no 0.5%-of-AGI charitable floor and guessed bracket tax well above the $5,778 the 2026 brackets give."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It got AGI of $162,672 by not subtracting the $3,088 of traditional 401(k) deferrals from wages, then added the educator expense back, pushing taxable income to $141,735 instead of $139,108. It then taxed that as if it reached the 9.3% bracket (about $7,973), when the 2026 MFJ brackets keep $139,108 in the 8% bracket at $5,778.21."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $138,000 was close, but it overstated MFJ bracket tax at about $7,700 instead of about $5,750. It then subtracted only $450 of exemption credits, even though its own components (two personal at about $150, two dependent at about $500) add up to about $1,300; the correct total is $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity, while the engine cuts the charitable deduction by 0.5% of AGI ($797.92) to $11,823.08. It also left the $338 educator expense out of CA AGI, which the engine subtracts. Together these put its taxable income at $138,647, $461 below the $139,108 that produces $5,778.21 of tax before the $1,284.46 of credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It overstated exemption credits at $1,800 by valuing all four exemptions at about $450. CA's personal exemption credit is only about $156 per spouse and the dependent credit about $486 each, for $1,284.46 in total. It also deducted the full $12,621 of charity with no 0.5%-of-AGI floor, giving $138,791 of taxable income instead of $139,108."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed about $150,000 of taxable income instead of $139,108 ($159,585 AGI less $20,477 itemized). It also reported its pre-credit figure unchanged and never subtracted the $1,284.46 of personal and dependent exemption credits it had listed."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a medical deduction to reach about $25.7k of itemized deductions. The two $2,500 premium fields are the same premium, so out-of-pocket medical is at most $11,400, below the 7.5%-of-AGI floor of about $11,969, and there is no medical deduction. Itemized deductions are $20,477 and taxable income is $139,108, not about $134k."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It started from an inflated CA AGI of $163,306 instead of $159,585, failing to fully apply the 401(k), IRA and $3,000 capital-loss adjustments. It added a $2,752 medical deduction even though medical costs are under the 7.5% floor. It also subtracted only about $1,000 of dependent credits and left out the two personal exemption credits (the full total is $1,284.46)."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $112,436 implies AGI of about $135,000, which means it subtracted the $24,597 of employer-sponsored insurance premiums from wages. The listed wages are already the taxable gross and CA AGI is $159,585, so it understated taxable income by about $26,700 against $139,108."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Taxable income of $107,524 after $27,545 of itemized deductions implies AGI of about $135,069, so it removed the $24,597 of employer-sponsored insurance premiums from income and added a medical deduction of about $6,270. Neither applies: AGI is $159,585 and itemized deductions are $20,477 (real estate taxes plus charity after the floor)."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $4,395.05 medical deduction from a $16,400 medical total that counts each adult's $2,500 premium twice. Correctly counted, out-of-pocket medical is at most $11,400, under the 7.5%-of-AGI floor of about $11,969, so the medical deduction is zero and taxable income is $139,108, not $134,396."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted about $1,180 of exemption credits instead of $1,284.46 for two personal and two dependent exemptions. Its implied pre-credit tax of about $5,845 is also above the $5,778.21 the 2026 MFJ brackets produce on $139,108 of taxable income."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $7,845 is about $3,350 too high. That gap matches overstated rates plus skipping the $1,284.46 of exemption credits. The correct steps are $159,585 AGI less $20,477 itemized = $139,108 taxable income, $5,778.21 of bracket tax, minus $1,284.46 of credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It cut CA AGI to $135,169 by removing the $24,597 of employer-sponsored insurance premiums from wages. It also added a medical deduction ($26,138 itemized instead of $20,477) and subtracted only about $600 of exemption credits instead of $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no figures. $2,570 plus the $1,284.46 of credits implies about $3,854 of bracket tax, which means taxable income near $114,000. That is about $25,000 below $139,108 and matches removing the $24,597 of employer-sponsored insurance premiums from wages."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $8,725 is more than the 2026 MFJ schedule charges even on the full $159,585 AGI with no deductions or credits (about $7,560), so it used inflated rates and gave no exemption credits. The correct result is $5,778.21 of tax on $139,108, minus $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,128 is $366 too low. At the 8% marginal rate that equals an extra deduction of about $4,500, the size of the medical deduction produced by counting the premium fields twice. In fact medical costs fall under the 7.5% floor and itemized deductions are $20,477."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,056 is $438 too low, which equals about $5,400 of extra deductions at 8%. That matches a medical deduction of about $4,400 (counting premiums twice) plus deducting full charity without the $797.92 floor. The correct itemized total is $20,477 and taxable income is $139,108."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $5,245 is $751 too high, which matches taking the $11,412 standard deduction instead of the $20,477 of itemized deductions. The extra $9,065 of taxable income at the 8% bracket adds about $725 of tax."
+us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used a made-up $6,566 CA standard deduction and wrongly decided itemizing did not pay, even though real estate taxes plus charity ($20,477) exceed the $11,412 standard deduction. It then subtracted the $338 educator expense as if it were a tax credit and skipped the $1,284.46 of exemption credits. Its reported $3,202.96 also contradicts its own $5,559."
+us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction of about $11,500 instead of itemizing $20,477, and subtracted only about $620 of exemption credits instead of $1,284.46. It also applied a CA child and dependent care credit, which CA only allows when AGI is $100,000 or less."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and $6,713 is $2,219 too high. That gap matches using the standard deduction instead of the $20,477 of itemized deductions and skipping the $1,284.46 of personal and dependent exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and $4,931 is $437 too high. The correct steps are $159,585 AGI less $20,477 itemized = $139,108 taxable income, $5,778.21 of 2026 MFJ bracket tax, minus $1,284.46 of exemption credits; its number reflects either too-high bracket tax or too-small credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It removed employer-sponsored insurance premiums from AGI ($134,323 instead of $159,585) and subtracted exemption deductions, but CA uses exemption credits, not deductions. It then reported $6,882 of tax on $101,811, more than double the roughly $3,100 the 2026 MFJ brackets charge on that amount."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the CA standard deduction instead of itemizing. The $20,477 of real estate taxes and charity (after the floor) beats the $11,412 standard deduction, so it overstated taxable income by about $9,000."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used $160,066 of AGI, leaving out the $144 IRA and $338 educator adjustments, and deducted the full $21,275 with no 0.5%-of-AGI charitable floor. Even on its own $138,791 of taxable income, 2026 brackets less $1,284.46 of credits give about $4,468, so its $4,382 also understates bracket tax."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no figures, and $4,250 is $244 too low, the same as about $3,000 of excess deductions at the 8% marginal rate. The correct steps are $139,108 of taxable income after $20,477 of itemized deductions, $5,778.21 of tax, minus $1,284.46 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 MFJ brackets without indexing ($5,871.38 on $138,791) and 2025 exemption credits ($1,256) instead of the 2026 values ($5,778.21 on $139,108 and $1,284.46 of credits). It also deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor and used $160,066 of AGI instead of $159,585."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the right kinds of deduction (charity and property tax) but came out $644 too high. Its bracket tax was above the $5,778.21 the 2026 MFJ brackets give on $139,108, and its exemption credits fell short of $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted full charity without the 0.5%-of-AGI floor, so its taxable income was $138,647 instead of $139,108, yet it still came out $72 high. That is because it applied bracket thresholds below the 2026 indexed values, overstating tax on that income."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied 2025 MFJ brackets without indexing ($5,871.38 on $138,791) and the 2025 exemption credit total ($1,256) instead of the 2026 values ($5,778.21 on $139,108 and $1,284.46). It also deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess ('~165k income approx 4.5k') with no deduction, bracket or credit work. The actual steps are $159,585 AGI, $20,477 itemized, $139,108 taxable, $5,778.21 of tax and $1,284.46 of exemption credits, giving $4,493.74, so the round number misses by $6.26."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $21,275 with no 0.5%-of-AGI charitable floor, then overstated bracket tax: $5,890 on $138,453, against $5,778.21 on the correct $139,108. That leaves it $106 too high even though its credit estimate of about $1,290 was close."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor, giving $138,791 of taxable income and $5,743 of tentative tax instead of $139,108 and $5,778.21. It also understated exemption credits at $1,246 instead of $1,284.46. The two errors nearly cancel, leaving it $3.26 too high."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor and used $159,766 of AGI, giving $138,491 of taxable income instead of $139,108. That $617 shortfall at the 8% rate, plus its bracket estimate, leaves the tax $67 too low."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It used $110,811 of taxable income, about $28,300 below the correct $139,108 ($159,585 AGI less $20,477 itemized). It also said no nonrefundable credits apply, but the $1,284.46 of personal and dependent exemption credits are available because their phase-out starts far above this AGI."
+us,scenario_099,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only $576 of exemption credits instead of $1,284.46 (two personal at about $156 and two dependent at about $486 each). That shortfall alone accounts for $708 of its $796 overstatement. It also deducted full charity without the 0.5%-of-AGI floor."
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score. The correct steps give $139,108 of taxable income, $5,778.21 of tax and $1,284.46 of exemption credits, for $4,493.74."
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score. The correct steps give $139,108 of taxable income, $5,778.21 of tax and $1,284.46 of exemption credits, for $4,493.74."
+us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported zero tax. CA taxes $139,108 of taxable income ($159,585 AGI less $20,477 itemized) at $5,778.21 before credits, and only $1,284.46 of nonrefundable exemption credits offset that."
+us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It assumed CA forces the standard deduction because the federal return takes it, but CA lets filers itemize on the state return on their own. The $20,477 of real estate taxes and charity (after the floor) beats the $11,412 standard deduction, so taxable income is $139,108, not about $147,800."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It skipped both the $3,000 capital-loss limit and the $3,088 401(k) exclusion in AGI and said no nonrefundable credits apply, leaving out $1,284.46 of exemption credits. It also badly understated bracket tax at $4,301.50 on $138,063, where the 2026 MFJ schedule gives about $5,695. The missing credits and the understated tax roughly cancel."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used a made-up $34,000 standard deduction, when CA's 2026 MFJ standard deduction is $11,412 and the $20,477 of itemized deductions applies. It also overstated tax at $8,100 on $130,894 (2026 brackets give about $5,120), applied a renter's credit that is not available at this AGI, and left out the $1,284.46 of exemption credits."
us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -7923,11 +8120,12 @@ us,scenario_100,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligi
us,scenario_100,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model collapsed Medicaid and CHIP into a single combined eligibility result, stating that the child qualifies for “Medicaid/CHIP.” The requested CHIP indicator is mutually exclusive with Medicaid eligibility here: Child 1 qualifies for Medicaid under the OLDER_CHILD category and is consequently ineligible for CHIP."
us,scenario_100,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not on Medicaid—but failed to evaluate it. Child 1 is Medicaid-eligible under Montana's OLDER_CHILD category, so the model's own stated CHIP condition is not satisfied."
us,scenario_100,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly placed the child below the Medicaid income limit but then mislabeled Medicaid coverage as “Medicaid-level CHIP coverage” and returned CHIP eligibility. PolicyEngine classifies Child 1 as Medicaid-eligible under the OLDER_CHILD category, and Medicaid eligibility categorically excludes the child from CHIP."
-us,scenario_100,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly treated age 6 as eligible merely because Head Start can serve children until compulsory school age. Under the benchmark's preschool-age test, Child 1 fails the age requirement, so the family's low income cannot produce eligibility."
-us,scenario_100,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model invented an unlisted circumstance in which the six-year-old had only recently turned five and was not yet in kindergarten. The prompt fixes the child's age at 6 for the full year and makes unlisted statuses false, so Child 1 does not satisfy the Head Start preschool-age test."
-us,scenario_100,child1_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model replaced the program's specific preschool-age requirement with the broader label ""preschool/early-school age."" A six-year-old fails the benchmark's Head Start age test regardless of low income."
-us,scenario_100,child1_head_start_eligible,grok-4.3,llm_error,age_disability,False,"The model asserted that age 6 met Head Start criteria without applying the preschool-age cutoff. Child 1 is outside the qualifying age range, so low household income does not establish eligibility."
-us,scenario_100,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model improperly extended the ordinary ages-3-to-5 rule through compulsory-school age and therefore counted the six-year-old as age-eligible. The benchmark's Head Start preschool-age test excludes Child 1, independently of the income result."
+us,scenario_100,child1_head_start_eligible,claude-haiku-5.5,llm_error,other,False,"The model's reasoning correctly found that Child 1, at age 6, is outside Head Start's preschool age range and concluded 'value = 0'. However, the value it submitted was Yes/1. The error is a mismatch between the submitted value and its own explanation, not a mistake in the policy reasoning."
+us,scenario_100,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model assumed Head Start serves children 'up to compulsory school age' and therefore let a 6-year-old qualify. In fact the preschool Head Start window ends before school age, so a 6-year-old is outside it. The model then treated income below the poverty line as decisive, even though Child 1 fails the age test whatever the household income."
+us,scenario_100,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model itself stated that Head Start serves ages 3-5, but then claimed a 6-year-old is 'still within the preschool-age range'. That contradicts the age limit it had just named, because age 6 is past the upper bound. It then decided the case on poverty-level income, which does not matter once the age test fails."
+us,scenario_100,child1_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model called age 6 'preschool/early-school age', which wrongly treated a school-age child as fitting Head Start's preschool window (age 3 up to school entry). Because of that, it approved eligibility on low income alone, even though Child 1 fails the age requirement."
+us,scenario_100,child1_head_start_eligible,grok-4.3,llm_error,age_disability,False,"The model said that age 6 plus low income 'meet Head Start criteria' without checking the preschool age limit. A 6-year-old is past the Head Start age window, so the child is ineligible regardless of income."
+us,scenario_100,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model stretched Head Start's ages 3-5 window to 'compulsory school age' so that a 6-year-old would count. It then treated income below 100% of the poverty line as enough for eligibility. But Child 1 fails the age test, so income is irrelevant. It also miscounted the household as 3 people when there are 4."
us,scenario_100,child1_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as a categorical Medicaid disqualifier. Montana's older-child Medicaid pathway applies a 143% FPL MAGI limit, and this child's 0.07 × FPL income satisfies it regardless of the listed employer-sponsored coverage."
us,scenario_100,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to apply Montana's older-child Medicaid category to the 6-year-old dependent. That pathway compares the child's 0.07 × FPL MAGI with the 143% FPL limit and yields eligibility.
us,scenario_100,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested output, violating the required output contract."
@@ -7946,57 +8144,60 @@ us,scenario_100,child2_head_start_eligible,gemini-3.5-flash-lite,llm_error,categ
us,scenario_100,child2_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model produced no derivation and defaulted the flag to 0 on the ground that eligibility was not affirmatively established, rather than applying Head Start's rules to the stated facts. Those facts decide it: Child 2 at age 5 is within the 3-through-compulsory-school-age band, TANF receipt of $505.41 monthly confers categorical eligibility on all household members, and the $1,875.40 AGI is far under the three-person poverty threshold — its 0 is consistent only with a blanket no-evidence-so-no rule applied to eligibility outputs."
us,scenario_100,child2_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"The model applied a truncated Head Start age band that excludes 5-year-olds, when the program covers ages 3 through compulsory school age and PolicyEngine treats ages 3 to 5 as preschool Head Start age (Early Head Start covers under 3, and Child 1 at 6 is the one above the band). The age screen short-circuited its answer before the household's TANF receipt of $505.41 monthly and its $1,875.40 AGI against the three-person poverty guideline could establish eligibility."
us,scenario_100,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Montana's YOUNG_CHILD Medicaid category to the 5-year-old dependent. At a MAGI income level of 0.07 times FPL, child2 satisfies the category's income test and is Medicaid eligible."
-us,scenario_100,child2_wic_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model incorrectly treated age 5 as within PolicyEngine's under-five WIC child category. It also presumed nutritional risk despite the instruction that unlisted statuses are false; low income does not cure the failed categorical test.
-us,scenario_100,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model interpreted the WIC child range as inclusive of age 5, but child eligibility ends at the fifth birthday. Its asset-limit analysis is irrelevant to this failed categorical test and also invents a WIC asset screen."
-us,scenario_100,child2_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model explicitly recognized that a child aged 5 has reached the fifth birthday and aged out, then reversed that conclusion because income was low. Income eligibility cannot override the under-five categorical requirement."
-us,scenario_100,child2_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly converted eligibility through the fifth birthday into eligibility after the child is already age 5. Certification-period continuation was not supplied and cannot be inferred under the prompt's constant-status and unlisted-facts rules.
-us,scenario_100,child2_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model called a five-year-old “under age 5” and therefore applied the WIC child category incorrectly. The income test is immaterial once the age-based categorical requirement fails.
-us,scenario_100,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 5 as included in the WIC child age range. PolicyEngine's categorical test requires the child to be under five, so the very low income does not produce eligibility."
-us,scenario_100,child2_wic_eligible,inkling,llm_error,categorical_eligibility,False,The model relied on age 5 and income below 185% of poverty without enforcing the under-five categorical requirement. A child already aged 5 is outside the WIC child category.
-us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child2_wic_eligible, violating the required output contract."
-us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model described a child aged 5 as “under 5,” which is the exact categorical error. Falling below the income threshold does not establish WIC eligibility for a child who has aged out."
-us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model interpreted “up to age 5” as including children who are already 5. WIC child categorical eligibility ends at the fifth birthday, regardless of the household's low income."
-us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model included age 5 in the WIC child category and then relied on the income limit. Because the child is not under five, the categorical test fails before income can establish eligibility."
-us,scenario_100,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"Its own reasoning reached the right total: EITC of 40% × $5,915 = $2,366 plus refundable CTC of 15% × ($5,915 − $2,500) = $512, which is $2,878. It then submitted 6,425, a number that no step in its reasoning supports. The error is in the value it submitted, not in the method."
-us,scenario_100,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It subtracted the $3,859 traditional 401(k) deferral from wages and used $2,056 as earned income. EITC earned income is actually gross wages of $5,915, so its EITC came out at $822.40 instead of $2,366. It also wrongly zeroed the refundable CTC, which is 15% × ($5,915 − $2,500) = $512."
-us,scenario_100,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It correctly worked out EITC of $2,366 and refundable CTC of 15% × $3,415 = $512.25. It then dropped the refundable CTC figure and swapped in an unexplained '$834 approximation', giving $3,200 instead of $2,878."
-us,scenario_100,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It computed the correct 40% × $5,915 = $2,366 EITC, then cut it to $1,906 by misapplying the rule that compares credits on AGI and on earned income. That comparison only applies once AGI reaches the phase-out range. In the phase-in range the credit is simply 40% of earned income, even when AGI is lower."
-us,scenario_100,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It derived EITC of $2,366 and refundable CTC of $512, which total $2,878. It then 'reduced slightly' the total to $2,330 on the mistaken idea that the 401(k) and IRA deductions trim the credits. In PolicyEngine, neither contribution reduces the earned income behind either credit."
-us,scenario_100,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It treated the $710 FLSA overtime premium as extra earnings on top of the $5,915 wages. The prompt says that wage total already includes overtime. It also inflated EITC to $2,905, more than 40% of even the double-counted $6,625. The correct figures are EITC of 40% × $5,915 = $2,366 plus $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It took earned income as wages net of the 401(k) deferral, $5,915 − $3,859 = $2,056. EITC earned income is actually the gross $5,915. As a result it produced $822.40 of EITC instead of $2,366 and wrongly zeroed the $512 refundable CTC."
-us,scenario_100,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It correctly computed EITC of $2,366 and refundable CTC of $512. It then threw that out and submitted $6,295, citing 'IRS EITC tables'. That figure is close to the maximum-credit amounts, which do not apply here: at $5,915 of earnings the household is in the 40% phase-in range, not at the plateau."
-us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It added the $710 FLSA overtime premium to the $5,915 wages to get $6,625 of earned income. The prompt states the wage total already includes overtime. The double count inflated EITC to $2,650 and refundable CTC to $618.75, instead of $2,366 and $512."
-us,scenario_100,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It subtracted the $3,859 traditional 401(k) deferral and both $564 health premium amounts from wages, leaving $928 of earned income. EITC earned income is gross wages of $5,915. That cut EITC to $371 instead of $2,366 and wrongly eliminated the $512 refundable CTC."
-us,scenario_100,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It reduced EITC earned income by the $3,859 traditional 401(k) deferral to $2,056. The correct base is gross wages of $5,915, so it got $822.40 of EITC instead of $2,366. It also dropped the $512 refundable CTC, which is 15% × ($5,915 − $2,500)."
-us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It got EITC of $2,366 right but used a $3,000 earnings threshold for the refundable CTC, assuming the pre-TCJA rules return in 2026. OBBBA made the $2,500 threshold permanent. The correct refundable CTC is 15% × ($5,915 − $2,500) = $512, not $437.25."
-us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It gave the household roughly the maximum two-child EITC plus a large refundable CTC, $6,848 in total. At $5,915 of earnings the EITC is still phasing in, at 40% × $5,915 = $2,366. The refundable CTC is limited to 15% × ($5,915 − $2,500) = $512."
-us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It treated the $20,644 of employer-sponsored insurance premiums as a pre-tax deduction that wiped out wages, and so used zero earned income. PolicyEngine does not reduce EITC or CTC earned income by those premiums; the base is the $5,915 of gross wages. That base gives $2,366 of EITC and $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It netted the health insurance premiums against wages to reach zero earned income and denied both credits. Earned income for the EITC and the refundable CTC is gross wages of $5,915. That yields $2,366 of EITC and $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned zero refundable credits with no working. A head-of-household filer with two qualifying children and $5,915 of earnings qualifies for 40% × $5,915 = $2,366 of EITC plus 15% × ($5,915 − $2,500) = $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It computed EITC correctly at $2,366 but used a $3,000 earnings threshold for the refundable CTC. The 2026 threshold is $2,500, made permanent by OBBBA. That is why it got $437.25 instead of $512 and a total of $2,803.25."
-us,scenario_100,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It submitted 0 even though its explanation concludes 'value = 2803.25', so the submitted value contradicts its own reasoning. That 2,803.25 also used the wrong $3,000 refundable CTC threshold instead of the 2026 $2,500 threshold."
-us,scenario_100,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It applied the pre-TCJA $3,000 earnings threshold, assuming the TCJA rules expire in 2026. OBBBA kept the $2,500 threshold permanently. Its refundable CTC was therefore $437.25 instead of $512, and its total $2,803.25 instead of $2,878."
-us,scenario_100,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It granted the full $1,700-per-child refundable cap, $3,400, and ignored the earned-income limit of 15% × ($5,915 − $2,500) = $512. It also estimated EITC at about $600 from $1,876 of AGI. The EITC phases in on earned income of $5,915, giving $2,366."
-us,scenario_100,federal_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It invented a guaranteed minimum refundable CTC of $1,650 per child, $3,300 in total, under OBBBA. The refundable portion is actually the lesser of the per-child cap and 15% of earnings above $2,500, with no floor. Here that is $512, which puts the total at $2,878, not $5,666."
-us,scenario_100,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It judged income too low for any refundable credit, but the EITC phases in from the first dollar of earnings. With $5,915 of earnings, EITC is 40% × $5,915 = $2,366. Refundable CTC is 15% × ($5,915 − $2,500) = $512."
-us,scenario_100,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It returned zero, missing that low earnings are exactly the EITC phase-in range. With two children and $5,915 of earnings, EITC is $2,366. Earnings above $2,500 also produce $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It based the EITC on $2,056 of wages after the $3,859 traditional 401(k) contribution instead of gross wages of $5,915. That produced $822.40 instead of $2,366. It also wrongly concluded earnings were below the refundable CTC threshold, losing the $512."
-us,scenario_100,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It subtracted the traditional 401(k) contribution from wages and used $2,056 as earned income. PolicyEngine uses gross wages of $5,915 for both credits. That base yields $2,366 of EITC and 15% × $3,415 = $512 of refundable CTC, not $822.40 and zero."
-us,scenario_100,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It claimed the phase-in amount was zero. The two-child EITC phases in at 40% of earned income from the first dollar, which on $5,915 gives $2,366. Refundable CTC adds another 15% × ($5,915 − $2,500) = $512."
-us,scenario_100,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It took EITC earned income as W-2 wages net of the 401(k) deferral, $2,056, instead of gross wages of $5,915, and so got $822 of EITC instead of $2,366. It also cited a $3,000 refundable CTC threshold instead of the 2026 $2,500, compounding its wrong zero for the $512 refundable CTC."
-us,scenario_100,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It computed EITC correctly at $2,366, but based the refundable CTC on a $3,000 earnings threshold. For 2026 the threshold is $2,500, so the refundable CTC is 15% × $3,415 = $512, not $437, and the total is $2,878."
-us,scenario_100,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It used W-2 Box 1 wages after the traditional 401(k) deferral, $2,056, as EITC earned income. PolicyEngine counts gross wages of $5,915. The result was $822 instead of $2,366 of EITC and $0 instead of $512 of refundable CTC."
-us,scenario_100,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It took earned income to be wages minus both the 401(k) deferral and the IRA contribution, $1,876, which is AGI rather than earned income. IRA contributions are above-the-line adjustments that never reduce earned income, and the base here is $5,915. That base gives $2,366 of EITC and $512 of refundable CTC, not $750 and zero."
-us,scenario_100,federal_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It equated EITC earned income with the $2,056 of W-2 Box 1 wages left after the 401(k) deferral, instead of the $5,915 of gross wages that PolicyEngine uses. That undercut EITC to $822.40 instead of $2,366 and wrongly eliminated the $512 refundable CTC."
-us,scenario_100,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"Its reasoning estimated EITC of about $6,500 and refundable CTC of about $2,300, yet it submitted 0, so the value contradicts its own analysis. Its estimates were also wrong: at $5,915 of earnings the EITC is in the 40% phase-in, at $2,366, and the refundable CTC is limited to $512."
-us,scenario_100,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It applied the 15% × ($5,915 − $2,500) earned-income formula once per child and got $1,024.50. That formula is a single family-level limit on the whole refundable CTC. The refundable CTC is $512 in total, making the total $2,878."
-us,scenario_100,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the CTC as refundable up to $1,800 per child, ignoring the earned-income limit of 15% × ($5,915 − $2,500) = $512. It then replaced its own correct $2,366 EITC with an arbitrary $3,730. Those two errors inflate the total to $7,330."
-us,scenario_100,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It claimed the EITC and refundable CTC are capped by tax liability. Refundable credits are paid out regardless of liability; that is what makes them refundable. The household receives $2,366 of EITC and $512 of refundable CTC."
+us,scenario_100,child2_wic_eligible,claude-fable-5,llm_error,age_disability,False,"The model stated the correct under-5 limit and then claimed that age 5 'falls within the eligible year' and is treated as the child category. WIC's child category ends at the 5th birthday, so a 5-year-old is categorically ineligible. The model also presumed nutritional risk and an income pass, but neither matters because the age test fails."
+us,scenario_100,child2_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated WIC's child range as 'ages 0-5' inclusive of age 5. The child category actually stops at the 5th birthday, so Child 2 is categorically ineligible. The model also invented an asset test that WIC doesn't have and counted 4 household members instead of 3, but the age error alone makes the answer wrong."
+us,scenario_100,child2_wic_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model called Child 2, age 5, 'under the WIC age limit of 5', which contradicts itself. A child who is 5 has reached the cutoff, so the strict under-5 test for WIC's child category fails and the child is ineligible."
+us,scenario_100,child2_wic_eligible,claude-opus-4.7,llm_error,age_disability,False,The model correctly noted that at age 5 the child has reached the 5th birthday and ages out of WIC. It then overrode that with an invented 'typical interpretation' that keeps age 5 eligible and answered 1. The correct conclusion is that Child 2 fails the categorical age test and is ineligible.
+us,scenario_100,child2_wic_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model said a 5-year-old 'still qualifies categorically as a child' because certification periods carry over. The child category is defined as under age 5, so a child who is already 5 is outside every WIC category. Meeting the 185% FPL income test doesn't change that."
+us,scenario_100,child2_wic_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model described age 5 as 'under age 5 cutoff', which is an incorrect comparison. Under WIC's strict under-5 test, Child 2 is outside the child category, so passing the income test doesn't make them eligible."
+us,scenario_100,child2_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model asserted that age 5 is within WIC's child age range. The range ends at the 5th birthday, so Child 2 is categorically ineligible no matter how low the household's income is."
+us,scenario_100,child2_wic_eligible,inkling,llm_error,age_disability,False,"The model checked only the 185% FPL income test and treated age 5 as meeting WIC's categorical requirement. WIC covers children only until their 5th birthday, so Child 2 is ineligible."
+us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model's answer for child2_wic_eligible was missing, with no value and no explanation, so there is nothing to score. The correct answer is 0 because Child 2 is age 5 and has aged out of WIC's under-5 child category."
+us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model said age 5 is 'within WIC's age limit of under 5', which contradicts itself. A 5-year-old fails the strict under-5 child category test and is ineligible despite the household's low income."
+us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model read WIC as covering 'children up to age 5' inclusive and counted the 5-year-old as eligible. The child category ends at the 5th birthday, so Child 2 is categorically ineligible."
+us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model placed age 5 within WIC's child age range. That range is under 5, so Child 2 fails the categorical test and is ineligible even though income is within limits."
+us,scenario_100,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"Its reasoning correctly reached an EITC of $2,366 (40% × $5,915) plus an ACTC of $512 (15% × $3,415), for $2,878. It then submitted $6,425, a number that matches neither its own arithmetic nor any derivation it stated."
+us,scenario_100,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It reduced EITC/ACTC earned income to $2,056 by subtracting the $3,859 traditional 401(k) deferral, which gave an EITC of 40% × $2,056 = $822.40 and no ACTC. The reference computes both credits on the full $5,915 gross wages: an EITC of $2,366 plus an ACTC of 15% × $3,415 = $512."
+us,scenario_100,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It correctly computed an EITC of $2,366 and an ACTC of 15% × ($5,915 − $2,500) = $512.25. It then replaced the ACTC with an unsupported $834 'approximation'. The refundable CTC is capped at the 15%-of-earnings-over-$2,500 amount, so the total is $2,878, not $3,200."
+us,scenario_100,federal_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It declined to compute the EITC or refundable CTC even though a head of household with two qualifying children and $5,915 of wages is in the EITC phase-in. That phase-in gives 40% × $5,915 = $2,366, and the ACTC is 15% × $3,415 = $512."
+us,scenario_100,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It computed an EITC of $2,366 and then cut it to an unexplained $1,906 by invoking a 'smaller of' AGI test. The AGI comparison only matters once income passes the phase-out threshold. In the phase-in range the credit is 40% of earned income, so the EITC stays at $2,366 and the total is $2,878."
+us,scenario_100,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It correctly identified an EITC of $2,366 and an ACTC of $512, which sum to $2,878. It then said the EITC was 'reduced slightly' with no rule behind the cut and submitted $2,330."
+us,scenario_100,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It added the $710 FLSA overtime premium to wages even though the prompt says gross wages already include overtime, so it counted the overtime twice. It then reported an EITC of $2,905 that matches no stated base. The correct EITC is 40% × $5,915 = $2,366, which with the $512 ACTC gives $2,878."
+us,scenario_100,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It used wages net of the $3,859 traditional 401(k) deferral ($2,056) as earned income, which gave an EITC of $822.40 and no ACTC. The reference uses the full $5,915 gross wages for both the EITC phase-in ($2,366) and the ACTC earnings formula ($512)."
+us,scenario_100,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It correctly derived an EITC of $2,366 plus an ACTC of $512. It then dropped that result and submitted $6,295 based on unspecified 'IRS EITC tables', which effectively credits a near-maximum EITC to a filer who is still in the 40% phase-in range."
+us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It treated earned income as $6,625 by adding the $710 FLSA overtime premium to the $5,915 gross wages, even though the prompt says gross wages already include overtime. This inflated the EITC to $2,650 and the ACTC to $618.75, instead of $2,366 and $512 on $5,915."
+us,scenario_100,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It cut earned income to $928 by subtracting the 401(k) deferral and both $564 health-premium entries as if they were pre-tax payroll deductions, then computed an EITC of $371 and no ACTC. The premiums are not salary reductions in the facts, and the reference applies both credits to the $5,915 gross wages."
+us,scenario_100,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It reduced earned income to $2,056 by subtracting the $3,859 traditional 401(k) deferral, which gave an EITC of $822.40 and no ACTC. The reference applies the 40% EITC phase-in and the 15%-over-$2,500 ACTC to the full $5,915 gross wages."
+us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It applied the pre-TCJA $3,000 ACTC earnings threshold on the assumption that TCJA expired in 2026, which gave an ACTC of $437.25. The OBBBA made the $2,500 threshold permanent, so the ACTC is 15% × $3,415 = $512 and the total is $2,878."
+us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It assumed the household gets the 'full' EITC (roughly the two-child maximum) plus refundable CTC. With $5,915 of earnings, the family is in the phase-in range, so the EITC is only 40% × $5,915 = $2,366 and the ACTC is $512."
+us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It treated the $20,644 employer-sponsored insurance premiums as pre-tax deductions that wipe out wages and set earned income to $0. The facts list no salary-reduction arrangement, and the reference computes the EITC ($2,366) and ACTC ($512) on $5,915 of wages."
+us,scenario_100,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It assumed pre-tax health insurance premiums reduce earned income to $0 and so denied the EITC and ACTC. The listed ESI premiums do not reduce wages, so the EITC is 40% × $5,915 = $2,366 and the ACTC is $512."
+us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned $0 without any computation, missing that a head of household with two qualifying children and $5,915 of wages gets a phase-in EITC of $2,366 and an ACTC of $512."
+us,scenario_100,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It computed the ACTC with the expired $3,000 earnings threshold, giving $437.25. For 2026 the threshold is $2,500 (made permanent by the OBBBA), so the ACTC is $512 and the total with the $2,366 EITC is $2,878."
+us,scenario_100,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It submitted $0 even though its own explanation concluded $2,803.25, so the submitted value contradicts its reasoning. The $2,803.25 itself relies on the expired $3,000 ACTC threshold instead of the $2,500 threshold that gives a $512 ACTC."
+us,scenario_100,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It assumed the TCJA had expired and used a $3,000 ACTC earnings threshold, giving $437.25. The OBBBA kept the threshold at $2,500, so the ACTC is 15% × $3,415 = $512 and the total is $2,878."
+us,scenario_100,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It granted the full $1,700-per-child refundable CTC ($3,400) without applying the limit of 15% × (earned income − $2,500) = $512. It also guessed an EITC of $600 from AGI instead of using 40% of the $5,915 earned income ($2,366)."
+us,scenario_100,federal_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It treated the per-child refundable cap as a floor by taking the maximum of $3,300 and the earnings-based $512. The ACTC is the lesser of the per-child cap and 15% × ($5,915 − $2,500), which is $512, so the total is $2,878, not $5,666."
+us,scenario_100,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It claimed the income was too low to produce refundable credits. In fact, $5,915 of earnings with two qualifying children produces a phase-in EITC of $2,366 and, because earnings exceed $2,500, an ACTC of $512."
+us,scenario_100,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It returned $0 and said no refundable credits are triggered. A head of household with two qualifying children and $5,915 of wages receives a phase-in EITC of $2,366 and an ACTC of $512."
+us,scenario_100,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It based the EITC on $2,056 of wages after the $3,859 traditional 401(k) deferral, giving $822.40 and no refundable CTC. The reference uses the $5,915 gross wages, which gives an EITC of $2,366 and an ACTC of $512."
+us,scenario_100,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It netted the traditional 401(k) deferral out of earned income ($2,056), which gave an EITC of $822.40 and put earnings below the ACTC threshold. The reference computes both credits on the $5,915 gross wages."
+us,scenario_100,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It asserted a zero EITC phase-in amount. A filer with $5,915 of earned income and two qualifying children gets 40% × $5,915 = $2,366 in EITC plus an ACTC of $512."
+us,scenario_100,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It took earned income as W-2 wages after the $3,859 401(k) deferral ($2,056), giving an EITC of $822 and no ACTC. The reference applies the EITC and the $2,500-threshold ACTC formula to the full $5,915."
+us,scenario_100,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used a $3,000 ACTC earnings threshold, giving $437. For 2026 the threshold is $2,500, so the ACTC is 15% × $3,415 = $512 and the total with the $2,366 EITC is $2,878."
+us,scenario_100,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It used W-2 Box 1 wages after the traditional 401(k) deferral ($2,056) as earned income, giving an EITC of $822 and no ACTC. The reference computes the EITC ($2,366) and ACTC ($512) on $5,915 gross wages."
+us,scenario_100,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It subtracted the traditional IRA contribution as well as the 401(k) deferral from earned income, leaving $1,876, and got an EITC of $750. An IRA contribution is an above-the-line AGI deduction and never reduces earned income. The reference computes both credits on $5,915, which gives $2,366 plus $512."
+us,scenario_100,federal_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It set earned income to $2,056 of W-2 Box 1 wages after the $3,859 401(k) deferral, giving an EITC of $822.40 and no refundable CTC. The reference uses the $5,915 gross wages, which gives an EITC of $2,366 and an ACTC of $512."
+us,scenario_100,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"Its reasoning estimated a large EITC (~$6,500) and an ACTC (~$2,300) but it submitted $0, contradicting its own analysis. The estimates were also wrong: in the phase-in the EITC is 40% × $5,915 = $2,366 and the ACTC is 15% × $3,415 = $512."
+us,scenario_100,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It multiplied the earnings-based ACTC amount by the number of children ($512.25 × 2). The limit of 15% × (earned income − $2,500) applies once per family, so the ACTC is $512 and the total is $2,878."
+us,scenario_100,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the CTC as refundable up to the full per-child cap ($3,600) without the 15%-of-earnings-over-$2,500 limit, which caps the ACTC at $512. It also inflated the EITC to an arbitrary $3,730 after correctly computing the 40% phase-in amount of $2,366."
+us,scenario_100,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It claimed the EITC and refundable CTC are capped by tax liability, but refundable credits are paid regardless of liability. The phase-in EITC of $2,366 and the earnings-based ACTC of $512 are both paid out in full."
us,scenario_100,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility with no derivation, never applying either free-tier pathway: SNAP receipt of $8,625.89 categorically confers free meals, and countable income of $11,979.68 (wages $5,914.72 plus TANF $6,064.96) is 44% of the three-person federal poverty guideline, far below the 130% free threshold. Its 0 is consistent with treating SNAP/TANF as absent because receipt is not a listed input fact, even though both are derived outputs of this same household. The income test alone settles it: the $5,915 of wages is about 22% of the poverty guideline, so the household lands in the free tier with two school-age children (ages 6 and 5) generating $7.15 per child per school day, $2,261.92 annually."
us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model gave a circular non-answer — that the benchmark does not indicate positive free-meal support — instead of computing the free-tier tests, so it applied neither SNAP categorical eligibility ($8,625.89 of SNAP automatically confers free meals) nor the 130%-of-poverty income test, which the household passes at 44% FPG on countable income of $11,979.68. Montana has no universal free-meals policy, so the tier turns entirely on those two tests, both of which this household clears. The correct derivation pays $7.15 per school day for each of the two school-age children (ages 6 and 5) for $2,261.92 annually; the model's 0 corresponds to no test being run at all."
-us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model evaluated the head only under an expansion/disability pathway and missed the parent/caretaker category, whose Montana income limit is 100% FPL and which the head satisfies at 0.07 × FPL. It also used an erroneous $3,627 annual 138%-FPL threshold and incorrectly imposed an asset limit on MAGI-based Medicaid eligibility."
-us,scenario_100,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the parent/caretaker pathway established by the head living with two dependent children. Under that category, the head’s MAGI of 0.07 × FPL is below Montana’s 100%-FPL limit, and neither separate disability-based expansion nor lack of other coverage is required."
-us,scenario_100,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for head_medicaid_eligible. The required result is 1 because the head qualifies under Montana’s parent/caretaker category at 0.07 × FPL, below the 100%-FPL threshold."
+us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It put 138% FPL for a household of three at about $3,627, when the real figure is about $36,000. That is roughly a tenfold error, and it led the model to conclude that $5,915 exceeded the limit, when $5,915 is only 7% of FPL. It also applied an asset test to the $2,800 in bank assets, but MAGI-category Medicaid (parent/caretaker or expansion adult) has no resource test."
+us,scenario_100,head_medicaid_eligible,claude-haiku-5.5,llm_error,health_coverage,False,"It treated the head's employer-sponsored insurance as disqualifying, but having ESI does not bar Medicaid eligibility. It also claimed the earnings fail Montana's income limits, when $5,915 is 7% of FPL for a household of three, well under the parent/caretaker limit."
+us,scenario_100,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It said the head's income level does not qualify and that no qualifying trigger applies. It missed that the head, as the parent of two dependent children, falls into Medicaid's parent/caretaker category, and that MAGI of 0.07 × FPL is far below that category's income limit."
+us,scenario_100,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no reasoning for head_medicaid_eligible, so there is no answer to score. The correct derivation puts the head in the parent/caretaker category with MAGI at 0.07 × FPL, which means eligible (1)."
us,scenario_100,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability as independently sufficient for Medicare eligibility at any age. It failed to require a Medicare-qualifying disability entitlement or another under-65 pathway, none of which is listed for the 46-year-old head."
us,scenario_100,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model invented a simplified PolicyEngine rule under which the disability flag automatically confers Medicare eligibility. PolicyEngine does not convert an unqualified disability input into Medicare eligibility without the required under-65 qualifying condition or entitlement.
us,scenario_100,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated a child's potential WIC eligibility as adjunctively conferring WIC eligibility on the household head. WIC does not extend individual eligibility to a parent or caretaker; the head is not stated to be pregnant, breastfeeding, or postpartum."
@@ -8013,52 +8214,53 @@ us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identi
us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($5,915 × 0.062 = $366.73 and $5,915 × 0.0145 = $85.78, totaling $452.51) and then discarded that result, submitting $477 on the invented rationale of 'rounding to annual benchmark.' No rounding, annualization, or state payroll-tax add-on applies here; the answer its own arithmetic supports is $452.48."
us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support."
us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals."
-us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It counted only the $493/month of wages and left out the $505.39/month of Montana TANF cash that SNAP counts as unearned income. It also took a $94 excess medical deduction, but the head is not SNAP-disabled because they receive no SSI or SSDI. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), and its $89 net income understates the $691 that correction gives (the frozen reference's is $237.09). Its own $758 × 12 is $9,096, not the $9,046 it submitted."
-us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It computed net income from wages alone, leaving out the ~$505/month of TANF cash, and subtracted a $1,124/year medical deduction the head doesn't qualify for (no SSI or SSDI, so not SNAP-disabled). It also used a $214 standard deduction instead of $209. It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. It then applied $785 × 12; holding the $785 maximum for October–December matches the release's SNAP convention (c_snap_hold_fy2026), and the frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule."
-us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It concluded the household was eligible but never applied the benefit formula (maximum allotment minus 30% of net income), and submitted an arbitrary 'conservative' $1,908 (~$159/month). The frozen reference's steps are $998.31 gross (wages plus TANF), minus the $209 standard, $98.58 earned-income and $453.64 excess shelter deductions, giving $237.09 net and $713.90/month for January–September. Its $453.64 comes from engine defect r30_snap_heat_and_eat_sua: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. The record's correction takes no utility-allowance shelter deduction and gives $691 net ($998.31 − $209 − $98.58, rounded), a $208 contribution and $577/month."
-us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It took a $141/month medical deduction for a head who receives no SSI or SSDI and so is not SNAP-disabled, and left out the ~$505/month of TANF income. It also used the FY2025 $768 maximum instead of $785. It then cut its own $9,108 result down to $7,320 because there was no shelter deduction. But a missing deduction can't lower a benefit already computed without it. Taking no shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 utility-allowance shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended."
-us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It used the outdated $766 maximum allotment instead of FY2026's $785 and worked from wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then reduced its own $8,496 to $8,316 for 'medical and shelter considerations', even though extra deductions raise the benefit rather than lower it."
-us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It counted only wages and applied only the standard and earned-income deductions, leaving out the $505.39/month of TANF income. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Its $8,300 (~$692/month) is below even its own wages-only formula ($785 − 30% × $185 ≈ $730). The frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It subtracted a $94/month disabled-member medical deduction, but the head receives no SSI or SSDI and so is not SNAP-disabled. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF from gross income. That produced $92 net income and $757.50/month for all 12 months, against the frozen reference's $237.09 net income, $713.90 for January–September and $733.60 for October–December; the $733.60 uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It used a $228 standard deduction instead of the FY2026 3-person $209, and took a $93.67 medical deduction for a head who is not SNAP-disabled (no SSI or SSDI). It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it counted wages only, leaving out $505.39/month of TANF. That gives $72.67 net instead of the $691 the correction gives (the frozen reference has $237.09), with a $784 maximum applied flat for 12 months."
-us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It treated the $20,644 employer-sponsored premium as an out-of-pocket medical deduction to push net income to zero, although the head isn't SNAP-disabled and that premium isn't a household expense. It then submitted $13,700, which is more than its own 12 × $766 = $9,192 ceiling. SNAP can never exceed the 3-person maximum allotment for the year."
-us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It used a disabled-member medical deduction to bring net income down to $0–$50, but the head receives no SSI or SSDI and gets no SNAP medical deduction. It also left out the $505.39/month of TANF income, which with no utility-allowance shelter deduction gives $691 net and a $208 monthly contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium on top of $5,915 in wages, although the wage total already includes overtime. It also subtracted a $100.67 medical deduction for a head who is not SNAP-disabled and left out the $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It used a $797 maximum and a $221 standard deduction instead of $785 and $209."
-us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It correctly counted TANF as unearned income, though at $458.54/month rather than $505.39, and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). Its net income came to $602, against $691 under that correction and the frozen reference's $237.09. It also took a $46.67 medical deduction for a head who is not SNAP-disabled and used the FY2025 $204 standard deduction and $768 maximum instead of $209 and $785."
-us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It submitted $9,037.20 while its explanation computes $7,808.48. That explanation understated TANF ($335/month vs $505.39) and took a $140.67 medical deduction for a head who is not SNAP-disabled. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also used a $198 standard deduction and $768 maximum instead of $209 and $785."
-us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"It took a $53.67 medical deduction for a head who receives no SSI or SSDI and so is not SNAP-disabled. It counted wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then applied an $805 maximum to all 12 months, although the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 $785 maximum for all 12 months of 2026."
-us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It deducted $140/month of the head's medical costs, but the head is not SNAP-disabled because they receive no SSI or SSDI. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out $505.39/month of TANF from gross income. That gives $54 net income instead of the $691 the correction gives (the frozen reference has $237.09), with a $781 maximum instead of $785."
-us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $9,408 is $784 × 12, a full maximum allotment with no household contribution. It ignored the 30% of net income that must be subtracted: $208/month on the $691 of net income from wages plus TANF under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10/month on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It said the household gets near the maximum allotment but submitted $6,128 ($510.67/month), which is more than $270/month below the $785 maximum and contradicts its own reasoning. The record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, gives $691 net income, a $208 contribution and $577/month; the frozen reference's $237.09 net income, $71.10 contribution and $713.90/month for January–September also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It counted TANF as unearned income and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended); its implied net income was ~$544 and its benefit $604.75/month. It also applied a medical deduction for a head who is not SNAP-disabled and used the FY2025 $768 maximum instead of $785."
-us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It said the household fails SNAP income and resource limits, but gross income is only 45% of the poverty guideline. The household is also broad-based categorically eligible through TANF non-cash status, and $2,800 in bank assets is within limits, so the benefit is positive."
-us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it applied a disabled-head medical deduction the head doesn't qualify for (no SSI or SSDI) and computed income from wages only, leaving out $505.39/month of TANF. The result was $753/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It submitted $9,216 ($768 × 12, the FY2025 maximum with no contribution) while its explanation says $9,072. It never subtracted 30% of net income: $691 from wages plus TANF after the standard and earned-income deductions under the record's correction of engine defect r30_snap_heat_and_eat_sua, or the frozen reference's $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used the FY2025 $204 standard deduction and $768 maximum. That gave $97 net income instead of the $691 the correction gives (the frozen reference has $237.09) and $739/month applied flat for 12 months."
-us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It counted the $20,644 employer-sponsored premium as a deductible medical expense of a disabled member to push net income to zero. The head isn't SNAP-disabled without SSI or SSDI, and SNAP income also includes $505.39/month of TANF, giving $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It used an $800 maximum instead of FY2026's $785."
-us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It correctly took no medical deduction and, as the record's correction of engine defect r30_snap_heat_and_eat_sua does, no utility-allowance shelter deduction (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF income and used a $215 standard deduction instead of $209. It also applied an $800 maximum to all 12 months instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; at $785 the frozen reference pays $713.90/month for January–September."
-us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"It said it used the 3-person maximum allotment, but $12,588 is $1,049/month, well above the FY2026 3-person maximum of $785 ($9,420/year). It also ignored the monthly contribution: $208 on $691 of net income under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10 on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It set SNAP to zero because no rent or utility costs were listed, but SNAP eligibility doesn't depend on shelter costs. This household has gross income at 45% of the poverty guideline, and the frozen reference's $237.09 net income gives $713.90–$733.60 per month; the $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It treated the head's health-insurance premiums, including the employer-sponsored plan, as a deductible medical expense of a disabled member to zero out net income. The head receives no SSI or SSDI and gets no SNAP medical deduction, and wages plus $505.39/month of TANF leave $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also used a $767 maximum instead of $785."
-us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It assumed countable net income was zero and paid the $785 maximum for all 12 months. It left out the $505.39/month of TANF that brings net income to $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, which requires a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It used the head's premiums and medical costs as a disabled-member deduction to wipe out net income, but the head receives no SSI or SSDI and is not SNAP-disabled. It also ignored $505.39/month of TANF income, so under the record's correction of engine defect r30_snap_heat_and_eat_sua net income is $691 and the benefit $577/month, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It counted SSI as unearned income, but SSI is zero because the head's $2,800 bank balance exceeds the $2,000 SSI resource limit. The actual unearned income is $505.39/month of TANF; the record's correction of engine defect r30_snap_heat_and_eat_sua takes no utility-allowance shelter deduction, while the frozen reference offsets the TANF with a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also applied a medical deduction the non-SNAP-disabled head doesn't qualify for, which put its $6,400 far below the frozen reference's $8,625.89, which uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 all year."
-us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It said the disabled head's insurance and medical expenses eliminate countable net income, but the head receives no SSI or SSDI and gets no SNAP medical deduction. With $505.39/month of TANF, net income is $691 and the contribution $208/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, so the benefit is below the $785 maximum; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"It computed income from wages only with no shelter costs, leaving out $505.39/month of TANF; taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua, since the frozen reference's $453.64 comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then applied a single $710.92 monthly amount to all 12 months; the frozen reference's rise to $733.60/month for October–December 2026 uses the engine's projected FY2027 parameters, which the release's SNAP convention (c_snap_hold_fy2026) replaces with the FY2026 schedule all year."
-us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted TANF cash as income and applied only the earned-income and standard deductions, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference also takes a $453.64 excess shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). The correction gives $691 net income and $577/month, $41 below its $618; at the $785 maximum, $618 implies about $555 of net income, so its estimated TANF was about $135/month below the $505.39. Measured against the frozen reference, leaving out the $453.64 deduction raised net income, and its benefit came to $618/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"It counted TANF income and explicitly assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance excess shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also took a medical deduction for a head who is not SNAP-disabled, and ended at $647/month instead of the frozen reference's $713.90 (January–September) and $733.60 (October–December); the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It returned $0 without doing any SNAP calculation, although the household has gross income at 45% of the poverty guideline and broad-based categorical eligibility; the record's correction of engine defect r30_snap_heat_and_eat_sua gives $691 net income and $577/month, and the frozen reference, whose $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), has $237.09 net income. That yields the frozen reference's $713.90–$733.60 per month; its $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted $1,688/year of medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used an outdated $198 standard deduction and $766 maximum instead of $209 and $785. That produced $668/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month)."
-us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It used an excess medical deduction for the 'disabled' head to bring net income to $0, but the head receives no SSI or SSDI and is not SNAP-disabled. With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also paid the outdated $766 maximum for all 12 months instead of $785 minus the $208 contribution under that correction (the frozen reference's is $71.10)."
-us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it subtracted a $1,688/year disabled-member medical deduction the head doesn't qualify for (no SSI or SSDI) and counted wages only, leaving out $505.39/month of TANF, which gave $536/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month). It also used a $787 maximum instead of FY2026's $785."
-us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"It reported $805, a single month's maximum allotment, as the annual benefit instead of adding up 12 monthly allotments. It also used a $182 medical deduction the non-SNAP-disabled head doesn't get to assume zero net income, when TANF-inclusive net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"It applied a disabled-member medical deduction that brought net income to ~$50, but the head receives no SSI or SSDI and is not SNAP-disabled. It also left out $505.39/month of TANF, which with wages gives $691 net and a $208 contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so there was no answer to score against the frozen reference's $8,625.89 annual benefit."
-us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"It deducted the head's premiums and medical expenses as a disabled-member medical deduction to set net income to zero, but the head receives no SSI or SSDI and is not SNAP-disabled. With $505.39/month of TANF, net income is $691, so the benefit is $577/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It subtracted the full ~$393 net income from the $766 maximum ($377/month) instead of 30% of net income. It also skipped the $209 standard deduction, left out TANF, and used an outdated $766 maximum instead of $785. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It deducted $182/month of medical costs, including both $564 premium entries and Child 2's $500, as a disabled-member medical deduction. The head receives no SSI or SSDI and is not SNAP-disabled, so there is no medical deduction. With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, not $3; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)."
-us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium to wages that already include overtime and used the outdated $766 maximum. It then submitted $12,444, which is more than its own 12 × $766 = $9,192 ceiling, even though it said it was adjusting the benefit downward."
-us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It said resources exceed SNAP limits, but the $2,800 bank balance is under the federal $3,000 resource limit. The household is also broad-based categorically eligible through TANF non-cash status, and vehicle value is excluded, so SNAP is positive, not zero: the frozen reference gives $8,625.89, using the engine's projected FY2027 schedule for October–December, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
+us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Used wages alone ($493 a month) and left out about $505 a month in Montana TANF cash, which counts as unearned income (gross is $998.31). It also missed the $453.64 shelter deduction from the utility allowance and took a $94 medical deduction the head does not qualify for, because the head receives no disability benefit. Its own figure, $758 × 12, is $9,096, not the $9,046 it submitted."
+us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted only the $5,915 in wages and left out Montana TANF cash, which counts as unearned income and brings gross to $998.31 a month. It said there were no shelter costs, which missed the $453.64 shelter deduction from the utility allowance. It also took a $1,124-a-year medical deduction, but the head receives no SSI or SSDI and so is not disabled for SNAP purposes."
+us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Agreed the household is eligible but never calculated a benefit and submitted an arbitrary 'conservative' $1,908. The correct calculation is $998.31 gross (wages plus TANF), minus $761.21 in deductions, for $237.09 net. That gives $713.90 a month for most of the year and $8,625.89 in total."
+us,scenario_100,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Returned $0, citing 'no benefit receipt inferred', even though the prompt says to assume take-up. The household passes every test: gross income is 45% of the poverty line, net income is 11%, and the $2,800 in assets is within the limit. It is owed $713.90–$733.60 a month."
+us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Started from wages only and left out Montana TANF cash (about $505 a month), which counts as unearned income. It took a $141 medical deduction that counted the $564 premium twice, even though the head is not disabled for SNAP purposes. It then cut its own $9,108 to $7,320 for a supposed lack of a shelter deduction, but the utility allowance actually produces a $453.64 shelter deduction."
+us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Counted wages only and left out about $505 a month in TANF cash. It also did not calculate the $453.64 shelter deduction from the utility allowance and used a low maximum allotment of $766. Its $8,316 is a hand adjustment of an earnings-only $8,496 estimate, not the calculation from $237.09 net income."
+us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Treated net income as small because it counted wages alone, leaving out Montana TANF cash, which counts as unearned income and brings gross to $998.31 a month. It also never calculated the $453.64 shelter deduction from the utility allowance. Its $8,300 is a rough guess, not $713.90 × 9 plus $733.60 × 3."
+us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Calculated net income of $92 from $493 in wages, leaving out about $505 a month in TANF cash. It took a $94 medical deduction even though the head receives no disability benefit and so is not disabled for SNAP purposes. It also missed the $453.64 shelter deduction from the utility allowance; the correct net income is $237.09."
+us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Set shelter costs to $0 because no rent was listed, but the Montana utility allowance produces a $453.64 shelter deduction. It left out TANF cash, which counts as unearned income (gross is $998.31). It also used a $228 standard deduction instead of $209 and took a $93.67 medical deduction the head does not qualify for, ending at a $762.20 monthly benefit instead of $713.90."
+us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Counted the $20,644 employer insurance premium as a deductible medical expense to bring net income to zero, and ignored TANF income. It then annualized a $766 monthly benefit as $13,700, more than 12 times any maximum allotment. The correct total is $8,625.89 ($713.90 or $733.60 a month)."
+us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Estimated net income at $0–50 by using wages alone and taking a medical deduction for the head. It left out about $505 a month in TANF cash, which counts as unearned income and raises net income to $237.09 even after the $453.64 shelter deduction. The head receives no SSI or SSDI, so there is no medical deduction."
+us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium on top of $5,915 in wages, but the prompt says wages already include overtime. It counted the $564 premium twice in a medical deduction the head does not qualify for. It also left out TANF cash and the $453.64 shelter deduction from the utility allowance."
+us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income, but at $458.54 instead of about $505.39. Its main error was leaving out the $453.64 shelter deduction from the Montana utility allowance, which left net income at $602.21 instead of $237.09. It also used a $204 standard deduction instead of $209, took a medical deduction the head does not qualify for, and used a $768 maximum instead of $785/$802.60."
+us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted TANF at only $335 a month instead of about $505. It left out the $453.64 shelter deduction from the utility allowance and took a $140.67 medical deduction the head does not qualify for, giving net income of $390.98 instead of $237.09. It then submitted $9,037.20, which contradicts its own stated result of $7,808.48."
+us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Calculated net income from wages alone, leaving out about $505 a month in TANF cash, which counts as unearned income. It took a $53.67 medical deduction even though the head receives no disability benefit, and missed the $453.64 shelter deduction from the utility allowance. It also used a $204 standard deduction instead of $209."
+us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Took a $140 medical deduction that counted the $564 premium twice ($2,108 a year), even though the head is not disabled for SNAP purposes. It ignored TANF cash, which counts as unearned income and brings gross to $998.31. It also left out the $453.64 shelter deduction from the utility allowance, ending at $54 net income instead of $237.09."
+us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $9,408 ($784 a month) assumes near-zero net income from wages alone. It leaves out about $505 a month in Montana TANF cash, which counts as unearned income. The correct net income is $237.09 and the household contribution is $71.10 a month."
+us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Said the household gets 'near the maximum' allotment but submitted $6,128 ($511 a month), which contradicts that claim and matches no calculation. The correct calculation counts TANF as unearned income and deducts $209 standard, $98.58 earned income and $453.64 shelter. That gives $713.90 a month."
+us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income, but left out the $453.64 shelter deduction from the Montana utility allowance. It took a medical deduction the head does not qualify for, because the head receives no disability benefit. It also used a $768 maximum instead of $785, giving about $605 a month instead of $713.90."
+us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Returned $0, saying the household fails the income and resource limits. Gross income is only 45% of the poverty line, net income is 11%, and the $2,800 in bank assets is within the limit. The household is owed $8,625.89."
+us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Started from $492.92 in monthly wages with no TANF cash, so it understated gross income by about $505 a month. It took a medical deduction even though the head receives no SSI or SSDI, and left out the $453.64 shelter deduction from the utility allowance. The result was about $753 a month instead of $713.90."
+us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Submitted $9,216 ($768 × 12), which is a full maximum benefit, while its explanation says 9,072. It never counted Montana TANF cash as unearned income, so it missed the $237.09 net income and the $71.10 monthly contribution."
+us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Calculated net income of about $97 from wages alone, leaving out about $505 a month in TANF cash, which counts as unearned income. It took a medical deduction the head does not qualify for and left out the $453.64 shelter deduction from the utility allowance. It also used outdated figures: a $204 standard deduction instead of $209 and a $768 maximum instead of $785."
+us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Deducted the $20,644 employer insurance premium and other medical costs to bring net income to zero. The head is not disabled for SNAP purposes (no SSI or SSDI), so no medical deduction applies. It also left out TANF cash, which counts as unearned income and keeps net income at $237.09."
+us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Calculated net income of $179.34 from wages alone, leaving out about $505 a month in TANF cash. It also left out the $453.64 shelter deduction from the Montana utility allowance. It applied a single $800 maximum for all 12 months instead of $785 for January–September and $802.60 for October–December."
+us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Its $12,588 ($1,049 a month) is above the maximum allotment for a three-person household ($785, rising to $802.60). It also ignored the $71.10 monthly contribution that comes from $237.09 net income once TANF cash is counted."
+us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned $0 because no rent or utility details were given, but SNAP does not require shelter costs. The Montana utility allowance still produces a $453.64 shelter deduction, and the eligible household is owed $8,625.89."
+us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Brought net income to zero with medical and premium deductions the head does not qualify for, because the head receives no disability benefit. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09. It also used a $767 maximum instead of $785/$802.60."
+us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Assumed zero net income and gave the full $785 × 12. It left out about $505 a month in Montana TANF cash, which counts as unearned income and creates a $71.10 monthly contribution. It also missed the October increase to $802.60."
+us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Brought net income to zero with medical deductions for the head, who receives no SSI or SSDI and so gets no medical deduction. It ignored TANF cash, which counts as unearned income and brings net income to $237.09. It then gave the full $785 × 12 instead of $713.90 for most months."
+us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI income the head does not receive: the $2,800 in bank assets exceeds the $2,000 SSI resource limit. It also took a medical deduction, but the unearned income is actually Montana TANF cash of about $505 a month. With the $453.64 shelter deduction, the correct benefit is $713.90 a month."
+us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Said the head's insurance and medical costs eliminate net income, but the head is not disabled for SNAP purposes (no disability benefit), so no medical deduction applies. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09, so it wrongly gave the full $785 × 12."
+us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Said there were no shelter costs, so it left out the $453.64 shelter deduction from the Montana utility allowance. It also left out TANF cash as unearned income. It applied one flat $710.92 for all 12 months, missing the October–December rise to $733.60 under the FY2027 maximum of $802.60."
+us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income but subtracted only the earned income and standard deductions. It left out the $453.64 shelter deduction from the Montana utility allowance, so net income was about $557 instead of $237.09 and the benefit $618 a month instead of $713.90."
+us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Counted TANF but explicitly applied 'no shelter deduction', missing the $453.64 shelter deduction from the Montana utility allowance. It also took a medical deduction the head does not qualify for, because the head receives no disability benefit. It used a flat $785 maximum for all 12 months, ignoring the October increase."
+us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Returned $0 without doing any calculation. The household passes the gross and net income tests and the asset test, with $237.09 net income. That gives $713.90 a month for nine months and $733.60 for three, or $8,625.89."
+us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Calculated net income of only $668 a year from wages alone, leaving out about $505 a month in TANF cash. It took a medical deduction that counted the $564 premium twice ($2,108 a year), even though the head is not disabled for SNAP purposes. It also left out the $453.64 shelter deduction and used an outdated $198 standard deduction and $766 maximum."
+us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Brought net income to zero with a medical deduction the head does not qualify for, because the head receives no SSI or SSDI. It left out TANF cash, which counts as unearned income and keeps net income at $237.09. It also used a $766 maximum instead of $785/$802.60."
+us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Took a $1,688 medical deduction that counted the $564 premium twice, but the head is not disabled for SNAP purposes, so no medical deduction applies. It also ignored TANF cash, which counts as unearned income (about $6,065 a year). It also left out the $453.64-a-month shelter deduction from the Montana utility allowance."
+us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Submitted one month's allotment ($805) instead of an annual total. It also brought net income to zero by ignoring TANF cash and taking a medical deduction the head does not qualify for. The correct benefit is $713.90–$733.60 a month, or $8,625.89 a year."
+us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"Estimated net income near $50 using wages alone plus a medical deduction for the head, who receives no disability benefit and so gets no medical deduction. It left out about $505 a month in TANF cash, which counts as unearned income, and never calculated the $453.64 shelter deduction. The correct net income is $237.09."
+us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was nothing to score against the $8,625.89 annual benefit."
+us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Brought net income to zero with premium and medical deductions the head does not qualify for, because the head receives no SSI or SSDI. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09 and a $71.10 monthly contribution. It also missed the October increase to $802.60."
+us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted all of its $393 net income from a $766 maximum instead of 30% of it, ending at $377 a month. It also skipped the $209 standard deduction, the $453.64 shelter deduction and TANF income."
+us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Took a $182 medical deduction based on $2,608 a year in costs, which included the child's $500 bill and the premium twice. The head is not disabled for SNAP purposes, so no medical deduction applies at all. It also left out TANF cash, which counts as unearned income, so net income was $3 instead of $237.09."
+us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium to wages even though the prompt says wages already include it. It then turned a $766 monthly figure into $12,444 a year, more than the $9,192 that 12 months at $766 would give. It also ignored TANF income and the $71.10 monthly contribution."
+us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Said the household's resources exceed SNAP limits, but the $2,800 in bank assets is within the limit, and the household is categorically eligible through TANF. It is owed $8,625.89."
us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head's generic disability fact as satisfying SSI's categorical disability requirement and proceeded directly to the earned-income calculation. PolicyEngine's SSI-specific aged/blind/disabled indicator is false for the head, so no SSI benefit-rate or income-exclusion calculation applies; its submitted value also contradicts its own stated $9,480 calculation."
us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the head's generic disability status with SSI categorical eligibility. The head's SSI-specific aged/blind/disabled indicator is false, so the earned-income reduction it applied is irrelevant and SSI is zero."
us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-qualified disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI's aged, blind, or qualifying-disabled criterion, so the calculation stops before income exclusions or the federal benefit rate are applied."
@@ -8068,178 +8270,183 @@ us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The
us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
us,scenario_100,ssi,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed the 46-year-old head's generic disability status met SSI's qualifying-disability test and then calculated countable wages. The SSI-specific categorical indicator is false, so the head is ineligible regardless of the wage calculation; neither child qualifies either."
us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the generic disability input as SSI categorical eligibility and therefore performed an income-reduction calculation that should never occur. It also added the separately listed overtime premium to annual gross wages and invented an IRWE adjustment despite instructions not to infer unlisted expenses, but categorical ineligibility alone makes SSI zero."
-us,scenario_100,state_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It used a 10% Montana EITC rate instead of the 2026 rate of 20%. It also switched between several earned-income bases and never settled on the gross $5,915 wage base; the correct federal EITC on that base is $2,365.89. Its final $265 is 10% of $2,646, a number with the 40% federal phase-in never applied, so both the federal base and the state rate are wrong."
-us,scenario_100,state_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed the federal EITC as $822.40, which is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The EITC base is gross earned wages of $5,915, which gives $2,365.89. It then applied the outdated 10% Montana rate instead of the 2026 rate of 20%."
-us,scenario_100,state_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It said Montana has no earned-income-based refundable credit. In fact Montana's refundable EITC equals 20% of the federal EITC in 2026. Being below the filing threshold does not block a refundable credit. The $2,365.89 federal EITC yields $473.18."
-us,scenario_100,state_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It correctly got a federal EITC of about $2,366 but applied a 10% Montana rate instead of the 2026 rate of 20%. It then submitted $297 even though its own 10% arithmetic gave $237. The correct amount is 20% × $2,365.89 = $473.18."
-us,scenario_100,state_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It recognized that the federal EITC is positive and that Montana's EITC is a percentage of it, but still reported $0 as a 'conservative' estimate. Montana's EITC is refundable and pays 20% of the $2,365.89 federal EITC, which is $473.18, even when state tax liability is zero."
-us,scenario_100,state_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It treated the Montana EITC as nonrefundable and zeroed it out because tax liability is zero. The Montana EITC is refundable and pays 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its federal EITC of $822.40 is 40% of wages after subtracting the traditional 401(k) deferral. The EITC base is gross wages of $5,915, which gives $2,365.89. It then applied the outdated 10% Montana rate instead of 20%, so it got $82.24 instead of $473.18."
-us,scenario_100,state_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It correctly computed a federal EITC of about $2,366 and correctly treated the Montana EITC as refundable. However, it applied the 10% rate enacted for 2024 instead of the 20% rate that applies in 2026, so it got $237 instead of $473.18."
-us,scenario_100,state_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It said the Montana EITC is nonrefundable and answered $0. The Montana EITC is refundable, at 20% of the $2,365.89 federal EITC in 2026, which is $473.18."
-us,scenario_100,state_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of about $2,366 but applied the old 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It applied a 12% Montana rate, which has never been Montana's EITC rate; the 2026 rate is 20%. It also used a $2,650 federal EITC, which does not match 40% × $5,915 = $2,365.89."
-us,scenario_100,state_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It computed a federal EITC of only $371, far below the two-child 40% phase-in result of $2,365.89 on $5,915 of earnings. It also applied a 10% Montana rate instead of 20%, so it got $37 instead of $473.18."
-us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It applied the repealed 3% Montana EITC rate to a federal EITC of $822.40, which was reduced for the 401(k) deferral. The correct figures are 20% of $2,365.89. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits. Its submitted $24.67 covers only the 3% EITC piece."
-us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It said no Montana refundable credits apply and overlooked Montana's refundable EITC. That credit pays 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It correctly identified the refundable Montana EITC and the $2,366 federal EITC but used the old 10% rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It found no state refundable credits and missed Montana's refundable EITC. That credit pays 20% of the $2,365.89 federal EITC earned on $5,915 of wages with two qualifying children."
-us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It said Montana has no refundable EITC and that the household has zero earned income. The head earns $5,915 in wages, which produces a $2,365.89 federal EITC. Montana's refundable EITC pays 20% of that, which is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It set the federal EITC to $0 and carried that zero into Montana. A head of household with two qualifying children under 19 and $5,915 of earned income is in the 40% phase-in, which gives $2,365.89. Montana's 20% EITC on that is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no reasoning and answered $0. The correct figure is 20% of the $2,365.89 federal EITC, which is $473.18. A $0 answer matches ignoring Montana's refundable EITC entirely."
-us,scenario_100,state_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"Its reasoning applied the repealed 3% Montana EITC rate and got $70.98. It then submitted $0, which contradicts that reasoning. The 2026 rate is 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It claimed PolicyEngine does not model Montana refundable credits. The Montana refundable EITC is computed at 20% of the federal EITC, which gives $473.18 here."
-us,scenario_100,state_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It claimed Montana has no state individual income tax. Montana levies an income tax and provides a refundable EITC equal to 20% of the federal EITC for 2026, which is $473.18 here."
-us,scenario_100,state_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of $2,366 but applied the original 3% Montana rate. That rate was raised to 10% for 2024 and to 20% for 2026, so it got $70.98 instead of $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It found no refundable Montana credits and overlooked the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It assumed a low-income household gets no Montana refundable credits. Low earned income is exactly what produces the federal EITC of $2,365.89, and Montana refunds 20% of it, which is $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It overstated the federal EITC at $4,207. The two-child 40% phase-in on $5,915 of earnings gives only $2,365.89. It also applied a 10% Montana rate instead of 20%."
-us,scenario_100,state_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of about $2,366 but the 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"It added a $1,200 young-child credit for the age-5 child, but this household's 2026 Montana refundable credits consist solely of the state EITC. It also applied a 10% EITC rate instead of 20%, getting $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It included a $1,200 young-child credit that is not part of this household's 2026 Montana refundable credits, which are only the state EITC. It also used a 10% rate on the $2,366 federal EITC instead of 20%, which would give $473.18."
-us,scenario_100,state_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It used the correct 20% Montana rate but applied it to an $822.40 federal EITC. That figure is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The federal EITC base is gross wages of $5,915, which gives $2,365.89, so the credit is $473.18."
-us,scenario_100,state_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"Its state EITC of $473.20 matches, but it added $1,800 of child tax credit for the two children. This household's 2026 Montana refundable credits consist solely of the state EITC, so the total inflated to $2,273.20."
-us,scenario_100,state_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It applied the correct 20% rate to an $822.40 federal EITC, which was computed on wages after subtracting the traditional 401(k) deferral. The EITC earned-income base is gross wages of $5,915, which gives a $2,365.89 federal EITC and a $473.18 Montana EITC."
-us,scenario_100,state_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said no Montana refundable credits apply and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It asserted that Montana offers no state EITC. Montana has a refundable EITC, 20% of the federal EITC for 2026, worth $473.18 here."
-us,scenario_100,state_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It stated Montana has no state EITC and looked only at elderly and property-tax credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It ruled out credits based on age and rent and claimed Montana has no EITC for this case. Montana's refundable EITC applies to any federal EITC recipient and pays 20% of $2,365.89, which is $473.18."
-us,scenario_100,state_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It said no refundable Montana credits apply and omitted the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It added a $1,200 child tax credit for the 5-year-old, but this household's 2026 Montana refundable credits are only the state EITC. It also used a federal EITC of about $750 instead of $2,365.89, and a 10% rate instead of 20%."
-us,scenario_100,state_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It stated Montana has no state EITC and looked only for rent or property-tax credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,state_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used the correct $2,366 federal EITC but the 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It claimed Montana has no state income tax. Montana taxes income and provides a refundable EITC equal to 20% of the federal EITC, which is $473.18 here."
-us,scenario_100,state_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It correctly took the federal EITC as $2,366 but applied the 10% rate that ended after 2025 instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18."
-us,scenario_100,state_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It asserted that Montana has no earned income tax credit and used zero tax liability as the reason for $0. Montana's EITC is refundable, pays 20% of the $2,365.89 federal EITC, and does not depend on tax liability."
-us,scenario_100,state_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It found no applicable Montana refundable credits and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18."
-us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found countable earnings of about $220 a month, below the payment standard, which makes the family eligible. It then reversed itself and submitted $0. The correct benefit is the $725.10 three-person payment standard minus $219.69, which is $505.41 a month or $6,064.96 a year."
-us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It claimed Montana TANF is not modeled and returned $0. In fact the family qualifies categorically with two minor children. Paying the $725.10 standard less $219.69 of countable earnings gives $6,064.96 a year."
-us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It sized the unit at four instead of three and applied a $2,000 asset limit. Montana's TANF resource limit is $3,000, so the $2,800 bank balance passes. It also compared gross wages to an income limit without the $200 + 25% earnings disregard, which leaves only $219.69 a month countable."
-us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"It asserted without calculating that countable income after disregards and a low payment standard would zero out the benefit. Countable earnings are only $219.69 a month against a $725.10 standard. Resources of $2,800 are under the $3,000 limit with the vehicle excluded, so the benefit is $505.41 a month."
-us,scenario_100,tanf,claude-opus-4.8,llm_error,other,False,"It assumed a roughly $588 maximum, then submitted $595 a month, which is above its own maximum, so it never subtracted countable earnings. The correct grant is the $725.10 standard minus $219.69 of countable earnings, which is $505.41 a month or $6,064.96 a year, not $7,140."
-us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It asserted PolicyEngine gives no Montana TANF benefit for this household and returned $0. Under the $725.10 standard, the family with two minor children and $219.69 of countable earnings receives $505.41 a month."
-us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"It used an estimated $730 payment standard and rounded countable earnings to $220. PolicyEngine's three-person standard is $725.10 and exact countable earnings are $219.69, giving $505.41 a month. That makes the correct annual amount $6,064.96, not $6,120."
-us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It applied a roughly $1,000 liquid-asset limit and ruled the family ineligible because of the $2,800 bank balance. Montana's TANF resource limit is $3,000, so the family passes. The benefit is $725.10 minus $219.69 of countable earnings."
-us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327 monthly payment standard and paid that full amount as the grant. The actual standard for three is $725.10. The correct benefit is that standard minus $219.69 of countable earnings, which is $505.41 a month or $6,064.96 a year."
-us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The family has two minor children, passes the $3,000 resource test, and receives the $725.10 standard less $219.69 of countable earnings."
-us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to the $2,800 bank balance, giving $4,781 of countable resources. Montana excludes the household vehicle, so countable resources are $2,800, which is under the $3,000 limit. The family is therefore eligible for $505.41 a month."
-us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $605 maximum grant and a $200 + 50% earnings disregard. Montana disregards $200 + 25%, which leaves $219.69 countable, not $146.46. The payment standard is $725.10, so the benefit is $505.41 a month, not $458.54."
-us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It submitted $0 even though its explanation computed $4,023.75. That computation also used a $555 payment standard instead of the $725.10 three-person standard. The correct benefit is $725.10 minus $219.69, which is $505.41 a month or $6,064.96 a year."
-us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 maximum and a $200 + 50% earnings disregard. Montana's standard for three is $725.10 and the disregard is $200 + 25%, which leaves $219.69 countable. The benefit is $505.41 a month, not $353.54."
-us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the correct $200 + 25% disregard, but used the outdated $588 payment standard instead of PolicyEngine's 2026 standard of $725.10. That understates the grant by about $137 a month: $368 instead of $505.41."
-us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared the household ineligible without naming any failed test. The family has two minor children, $2,800 of resources under the $3,000 limit, and $219.69 of countable earnings against a $725.10 standard. That qualifies it for $505.41 a month."
-us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It assumed the head receives SSI and zeroed TANF on that basis. SSI is $0 because the $2,800 bank balance exceeds SSI's $2,000 resource limit, so no SSI offset or exclusion applies. The family receives the $725.10 standard minus $219.69."
-us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $5,298 answer matches a $588 payment standard with a $200 + 50% disregard. Montana disregards $200 + 25%, leaving $219.69 countable, and the 2026 standard for three is $725.10. The correct benefit is $505.41 a month."
-us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without reasoning, which treats the family as ineligible. Two minor children, resources under $3,000, and $219.69 of countable earnings against the $725.10 standard produce $6,064.96 a year."
-us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $5,298.48 equals $588 minus $146.46 times 12, which is an outdated $588 standard with a $200 + 50% disregard. The correct figures are a $725.10 standard and a $200 + 25% disregard ($219.69 countable), giving $505.41 a month."
-us,scenario_100,tanf,gemini-3.7-flash,llm_error,thresholds_rates,False,"It submitted $0 while its explanation said the household qualifies and computed $4,824. That $402-a-month figure is also about $103 short of the correct $505.41, which is the $725.10 standard less $219.69 of countable earnings."
-us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It asserted Montana TANF is not modeled and returned $0. The eligible three-person family receives the $725.10 standard minus $219.69 of countable earnings, which is $6,064.96 a year."
-us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $710 FLSA overtime premium on top of wages that already include overtime. It then compared the full $6,625 to the payment standard without the $200 + 25% earnings disregard, and invoked a work requirement that 55 hours a week satisfies. Correct countable earnings are $219.69 a month against a $725.10 standard."
-us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,"It asserted PolicyEngine gives no positive Montana TANF benefit here. The family passes the categorical, $3,000 resource, and income tests, and receives $725.10 minus $219.69, which is $505.41 a month."
-us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated TANF as not payable because receipt was not listed, but the prompt assumes take-up. The family with two minor children receives the $725.10 standard less $219.69 of countable earnings, which is $6,064.96 a year."
-us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,It claimed no TANF-qualifying circumstances exist. Two dependent children aged 6 and 5 in a low-earning single-parent household satisfy Montana's categorical requirement. The benefit is $505.41 a month.
-us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"It estimated $150 of countable income, which is consistent with a 50% disregard, and a grant of $438. Montana disregards $200 + 25%, which leaves $219.69 countable, and the three-person standard is $725.10. The correct benefit is $505.41 a month."
-us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50 monthly grant reflects a payment standard near the outdated $588. The 2026 three-person standard is $725.10, and subtracting $219.69 of countable earnings gives $505.41 a month."
-us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,419.75 equals $588 minus $219.69 times 12, so it used the outdated $588 payment standard. The 2026 standard is $725.10, which yields $6,064.96 a year."
-us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It counted SSI income against TANF. SSI is $0 because the $2,800 bank balance exceeds SSI's $2,000 resource limit. The only countable income is $219.69 of earnings against a $725.10 standard."
-us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"It used $725 as the three-person payment standard instead of the exact $725.10 ($8,701.21 a year). That understated the annual benefit by $1.21: $6,063.75 instead of $6,064.96."
-us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,298 matches a $588 standard with a $200 + 50% disregard. Montana disregards $200 + 25%, leaving $219.69 countable, and the 2026 standard is $725.10. The correct benefit is $505.41 a month."
-us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25 monthly grant is $588 minus $219.69, so it used the outdated $588 payment standard rather than the 2026 standard of $725.10. The correct grant is $505.41 a month."
-us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,"It applied the correct $200 + 25% disregard but used a $588 three-person payment standard instead of $725.10. That understates the benefit by about $1,645 a year."
-us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It computed nothing and returned $0. The eligible family receives the $725.10 standard minus $219.69 of countable earnings, which is $6,064.96 a year."
-us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"It used the outdated $588 payment standard. With the $725.10 standard for 2026, $725.10 minus $219.69 of countable earnings is $505.41 a month, not $368."
-us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It used a $588 payment standard instead of $725.10. With $219.69 of countable earnings the grant is $505.41 a month, not $368."
-us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"It got the countable earnings exactly right at $219.69, but subtracted them from an outdated $588 standard instead of $725.10. That produced $4,419.75 instead of $6,064.96."
-us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,It used a $675 maximum for a family of three instead of the $725.10 standard. That left a $455 grant in place of $505.41 a month.
-us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It used the outdated $588 maximum grant rather than the 2026 three-person standard of $725.10. After subtracting $219.69 of countable earnings, the correct grant is $505.41 a month, not about $368."
-us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no TANF value or explanation, so there was no answer to score against the $6,064.96 reference."
-us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"It handled the resource test and the $200 + 25% disregard correctly ($219.69 countable), but used a $588 grant level instead of the $725.10 three-person standard. The result understates the annual benefit by $1,645.21."
-us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana grants are only $50 to $200 a month and that earnings wipe them out. The three-person standard is $725.10, so $219.69 of countable earnings leaves a $505.41 monthly grant."
-us,scenario_100,tanf,ox-alpha,llm_error,categorical_eligibility,False,"It imputed about $780 a month of SSI as unearned income. SSI is $0 because the $2,800 bank balance exceeds the $2,000 SSI resource limit, so the only countable income is $219.69 of earnings against a $725.10 standard."
-us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,It added the $710 overtime premium to wages that already include it. It then compared gross annual income to the payment standard without applying the $200 + 25% monthly earnings disregard. Countable earnings are only $219.69 a month against a $725.10 standard.
-us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"It declared income too high without applying the $200 + 25% disregard. That disregard reduces $492.92 of monthly wages to $219.69 countable, well below the $725.10 standard, so the benefit is $505.41 a month."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the full $1,400 of auto loan interest as an OBBBA qualified vehicle-loan deduction. No qualifying vehicle is established, so taxable income is $85,030, not $83,630. It then submitted $12,064, which matches none of its own computations ($13,111 or $13,058), and in the second pass it swapped in wrong bracket thresholds ($12,525/$50,900) for the correct $12,400/$50,400."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted $1,000 of auto loan interest, taking the $400 phaseout off the $1,400 of interest rather than off the $10,000 cap. The listed interest is not qualified passenger-vehicle loan interest, so no such deduction applies. Its brackets were correct, but taxable income of $84,030 instead of $85,030 understates tax by 22% x $1,000 = $220."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It added the nontaxable $700 of veterans benefits to AGI and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It taxed everything above the 10% band at 12%, ignoring the 22% bracket above $50,400. It then submitted $11,435, which does not even match its own $10,637.50 figure."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived the exact correct figure: $85,030 taxable income and $1,240 + $4,560 + $7,618.60 = $13,418.60. It then discarded that result and submitted $14,716 under the label 'rounding to bracket estimates', which no computation supports."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the correct taxable income of $85,030 and the correct bracket tax of about $13,419. It then submitted $13,876, which none of its stated bracket arithmetic supports."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the full $1,400 of auto loan interest as an OBBBA qualified vehicle-loan deduction. It applied the $400 phaseout correctly against the $10,000 cap, but the listed interest is not established as qualified new, U.S.-assembled vehicle loan interest. Taxable income is therefore $85,030, not $83,630, and the extra deduction cut tax by 22% x $1,400 = $308."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on a $15,750 standard deduction (the 2025 amount) instead of the 2026 amount of $16,100. It also used invented brackets (10% to $12,300, 12% to $49,950) in place of 10% to $12,400 and 12% to $50,400. That yields $85,380 of taxable income and $13,543 of tax, instead of $85,030 and $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used understated bracket thresholds of $12,150 and $49,400 instead of $12,400 and $50,400, producing $13,523.60. It then submitted an unsupported $13,322 'after adjusting for standard deduction indexing', although it had already applied the deduction."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It took a $1,174 auto-loan-interest deduction, reducing the $1,400 of interest by a phaseout. The listed interest is not qualified passenger-vehicle loan interest, so no deduction applies, and taxable income is $85,030 rather than $83,856. That understated tax by roughly 22% x $1,174."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,600 standard deduction instead of $16,100, and brackets of 10% to $12,175 and 12% to $49,550 instead of $12,400 and $50,400. That inflated taxable income to $85,530 and tax to $13,618."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $3,840 of health premiums from wages as a pre-tax exclusion by double-counting the two $1,920 premium fields. The $103,000 is gross taxable wages and no premium exclusion applies. It also used a $15,400 standard deduction and $12,200/$49,700 brackets instead of $16,100 and $12,400/$50,400."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied an $8,300 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption and 10/12/22% brackets."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of the 2026 amount of $16,100, and brackets of 10% to $12,200 and 12% to $49,575 instead of $12,400 and $50,400. That produced $85,780 of taxable income and $13,670.10 of tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,300 standard deduction, a $5,300 personal exemption and 10/15/25% brackets instead of OBBBA's permanent $16,100 standard deduction and 10/12/22% brackets. It also wrongly subtracted $1,920 of premiums from the gross taxable wages."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It started from $103,000 of wages without subtracting the $1,870 capital loss, and it used a $15,300 standard deduction instead of $16,100. That gave $87,700 of taxable income instead of $85,030. Its $13,385 also does not follow from the correct 2026 brackets on its own taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It stated the correct AGI of $101,130 but never applied the $16,100 standard deduction or the 10%/$12,400, 12%/$50,400, 22% brackets. It submitted a round guess of $15,400 instead of computing $13,418.60 on $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used an $8,250 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA permanent: the $16,100 standard deduction applies, there is no personal exemption, and 10/12/22% rates apply."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It cited only a vague standard deduction of about $15,000+ and never applied the 2026 figures ($16,100 deduction; 10% to $12,400, 12% to $50,400, then 22%). Its $11,102 corresponds to roughly $74,500 of taxable income, far below the correct $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired and applied an $8,300 standard deduction, a $5,300 personal exemption and outdated 10/15/25% brackets ($11,000/$44,700). OBBBA made the $16,100 standard deduction and the 10/12/22% brackets permanent."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and subtracted an $8,300 standard deduction plus a $5,300 personal exemption, then applied pre-TCJA rates. Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets apply."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket thresholds ($11,925/$48,475). The correct 2026 figures are a $16,100 deduction and $12,400/$50,400 thresholds, which give $85,030 of taxable income instead of $86,130."
-us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $15,450 standard deduction and self-inflated brackets ($12,283/$49,929) instead of the 2026 figures of $16,100 and $12,400/$50,400. That overstated taxable income at $85,680 and gave tax of $13,611.04."
-us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its brackets were correct, but it used a $15,800 standard deduction instead of the 2026 amount of $16,100. That overstated taxable income by $300 ($85,330 vs $85,030) and tax by 22% x $300 = $66."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated the tax-exempt $2,308 pension as taxable and described 'small taxable capital gains' where there is a $1,870 net loss that reduces AGI to $101,130. That inflated taxable income above $85,030 and produced $14,498."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It invoked itemized medical adjustments that do not apply: $2,520 of medical costs sit far below the 7.5%-of-AGI floor, and the $16,100 standard deduction exceeds itemized deductions. Its $9,587 corresponds to taxable income around $67,000, far below the correct $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the scheduled TCJA-sunset parameters (a small standard deduction plus a personal exemption, about $87,380 of taxable income, pre-TCJA rates). Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets give $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted a phased-down $1,000 qualified auto-loan-interest deduction, reaching $84,030 of taxable income. The listed auto loan interest is not qualified passenger-vehicle loan interest, so taxable income stays $85,030, and its answer is $220 (22% x $1,000) too low."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,000 auto-loan-interest deduction after phaseout. No qualifying new, U.S.-assembled vehicle loan is established, so no OBBBA vehicle-interest deduction applies. Taxable income is $85,030, not $84,030, and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 amount of $16,100, giving $85,830 of taxable income. It then applied unspecified 'projected' brackets that overshoot even that base: $13,774 versus $13,594.60 under the correct 10%/$12,400, 12%/$50,400, 22% schedule."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026 and used an $8,400 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket thresholds ($11,925/$48,475) instead of the 2026 figures of $16,100 and $12,400/$50,400. That gave $86,130 of taxable income and $13,863 of tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used bracket thresholds of $12,200 and $49,575 instead of the 2026 thresholds of $12,400 and $50,400. Too much income fell in the 22% band, producing $13,505.10."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted TCJA law' with an $8,300 standard deduction, a $5,300 exemption and 10/15/25% brackets. Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets apply to $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but reported $13,426 instead of the exact bracket result $1,240 + $4,560 + $7,618.60 = $13,418.60. The $7.40 overstatement comes from bracket thresholds slightly off the 2026 $12,400/$50,400 schedule."
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $13,418.60 of tax on $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,174 auto-loan-interest deduction, reducing the $1,400 of interest by 20% of AGI over $100,000. The listed interest is not qualified passenger-vehicle loan interest, so no deduction applies, and taxable income is $85,030 rather than $83,856."
-us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Its own reasoning acknowledged positive tax on $103,000 of wages after the standard deduction, yet it submitted $0 by deferring to 'the numeric answer returned'. It never computed the $13,418.60 bracket tax on $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction instead of $16,100 and brackets of $12,150/$49,000 instead of $12,400/$50,400, which gave $13,641. It then submitted $14,827, which contradicts that computation."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It ignored the $1,870 capital-loss deduction and used a $13,600 standard deduction instead of $16,100. It taxed everything above $12,400 at 12%, ignoring the 22% bracket above $50,400. It then subtracted $4,824 of payroll tax as a nonrefundable credit, but no such credit exists."
+us,scenario_100,state_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate instead of the 2026 rate of 20%. It also guessed earned income at about $2,646 after a 401(k) adjustment. The federal EITC is 40% of the full $5,915 in wages, or $2,365.89, so the Montana credit is $473.18; the model's confused math produced $265."
+us,scenario_100,state_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"The model subtracted the $3,859 traditional 401(k) deferral from wages, giving a $822.40 federal EITC (40% × $2,056). The correct federal EITC is $2,365.89 on the full $5,915. The model then applied a 10% Montana rate instead of the 2026 rate of 20%."
+us,scenario_100,state_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"The model said Montana has no refundable earned income credit. Montana's EITC is refundable and pays 20% of the federal EITC in 2026, which is $473.18 on a $2,365.89 federal EITC. The model also wrongly used being below the filing threshold as a reason for $0, but a refundable credit is paid even with no tax liability."
+us,scenario_100,state_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"The model gave no derivation and left out the Montana refundable EITC completely. The correct amount is 20% of the $2,365.89 federal EITC (40% of $5,915 with two qualifying children), or $473.18."
+us,scenario_100,state_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"The model computed the federal EITC correctly at about $2,366, then applied a 10% Montana rate instead of the 2026 rate of 20%. It also turned its own $237 result into $297 with no basis. The correct credit is 20% × $2,365.89 = $473.18."
+us,scenario_100,state_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"The model said a positive federal EITC and a Montana state EITC both applied, then reported $0 anyway as a 'conservative' estimate. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18, and is paid even with no tax liability."
+us,scenario_100,state_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"The model treated the Montana EITC as nonrefundable, so zero tax liability wiped it out. The Montana EITC is refundable, and at 20% of the $2,365.89 federal EITC it pays $473.18 regardless of tax owed."
+us,scenario_100,state_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"The model based the federal EITC on wages minus the traditional 401(k) deferral, giving $822.40, instead of the full $5,915, which gives $2,365.89. It also applied 10% instead of Montana's 2026 rate of 20%."
+us,scenario_100,state_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"The model correctly found a refundable Montana EITC on a federal EITC of about $2,366, but used a 10% rate. Montana's 2026 rate is 20%, so the credit is $473.18, not $237."
+us,scenario_100,state_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"The model said the Montana EITC is nonrefundable and set refundable credits to $0. The Montana EITC is refundable and equals 20% of the $2,365.89 federal EITC, which is $473.18."
+us,scenario_100,state_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"The model used the right federal EITC base (about $2,366) but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18, not $236.60."
+us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"The model used a 12% Montana EITC rate that does not exist instead of the 2026 rate of 20%. It also overstated the federal EITC at $2,650; the correct figure is 40% × $5,915 = $2,365.89. The correct Montana credit is $473.18."
+us,scenario_100,state_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at $371, as if most of it had phased out, but at $5,915 of earnings with two children the credit is still in the 40% phase-in range and equals $2,365.89. The model also used a 10% Montana rate instead of 20%. The correct Montana credit is $473.18."
+us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"The model used the outdated 3% Montana EITC rate on a federal EITC that had been reduced by the 401(k) deferral to $822.40. The correct figure is 20% of $2,365.89. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits, and the value it submitted ($24.67) does not match its stated total ($1,224.67)."
+us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"The model said Montana has no refundable credits for this household and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18, not $236.60."
+us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"The model claimed the household has zero earned income and that Montana has no refundable EITC. Both claims are wrong: the head earns $5,915, which produces a $2,365.89 federal EITC, and Montana's refundable EITC pays 20% of that, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"The model set the federal EITC to $0, which also zeroed out the Montana EITC. A head of household with $5,915 of earnings and two qualifying children (ages 6 and 5) gets 40% × $5,915 = $2,365.89, so the Montana credit is $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"The model gave no derivation and left out the Montana refundable EITC. The correct amount is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"The model's reasoning used the outdated 3% Montana EITC rate to get $70.98, yet it submitted $0, which contradicts its own explanation. The 2026 rate is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"The model claimed PolicyEngine does not model Montana refundable credits. The Montana EITC is modeled and refundable at 20% of the $2,365.89 federal EITC, which gives $473.18."
+us,scenario_100,state_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state income tax. Montana does levy one, and it includes a refundable EITC equal to 20% of the federal EITC, or $473.18 here."
+us,scenario_100,state_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but the 3% Montana rate from 2024. The 2026 rate is 20%, which gives $473.18."
+us,scenario_100,state_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"The model found no Montana refundable credits and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"The model assumed a low-income household gets no Montana refundable credit. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at $4,207, which is more than 40% × $5,915 = $2,365.89 allows in the phase-in range. It also used a 10% Montana rate instead of 20%. The correct Montana credit is $473.18."
+us,scenario_100,state_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18."
+us,scenario_100,state_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate ($236.60) instead of 20% ($473.18). It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits; the EITC alone is the total."
+us,scenario_100,state_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate ($236.60) instead of 20% ($473.18). It also added a $1,200 young-child credit for the five-year-old, but this household's Montana refundable credits consist only of the EITC."
+us,scenario_100,state_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"The model used the correct 20% Montana rate but a federal EITC of $822.40. That figure is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The engine uses the full $5,915 of earned income, giving a $2,365.89 federal EITC and a $473.18 Montana credit."
+us,scenario_100,state_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"The model got the Montana EITC right at $473.20 (20% of the federal EITC). It then added $1,800 of Montana child tax credit, but this household's Montana refundable credits are the EITC alone."
+us,scenario_100,state_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"The model used the correct 20% rate but on an $822.40 federal EITC, which came from cutting earned income by the traditional 401(k) deferral. With the full $5,915 of earned income, the federal EITC is $2,365.89 and the Montana credit is $473.18."
+us,scenario_100,state_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state EITC. Montana has a refundable EITC that is 20% of the federal EITC in 2026, or $473.18 here."
+us,scenario_100,state_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state EITC and only checked property-tax-style credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"The model ruled out the elderly homeowner/renter credit correctly but wrongly said Montana has no state EITC. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and left out the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at about $750 instead of $2,365.89 and used a 10% Montana rate instead of 20%. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits. The correct total is the EITC alone, $473.18."
+us,scenario_100,state_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no refundable state EITC. Montana's refundable EITC equals 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18."
+us,scenario_100,state_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state income tax. Montana does levy one, and it includes a refundable EITC equal to 20% of the federal EITC, or $473.18 here."
+us,scenario_100,state_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18."
+us,scenario_100,state_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no earned income tax credit. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, or $473.18, even with no Montana tax liability."
+us,scenario_100,state_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,tanf,claude-fable-5,llm_error,categorical_eligibility,False,"Its own reasoning found the family eligible, with countable income of about $220 a month and a positive benefit of about $368 a month, but it then submitted $0. It also used a $588–$633 payment standard instead of Montana's 2026 three-person standard of about $725 a month, which leaves a benefit of about $505 a month."
+us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It wrongly said Montana TANF is not modeled. The program is modeled, and this family qualifies: the about $725 payment standard minus $219.69 in countable earnings (after the $200 + 25% disregard) gives about $505 a month, or $6,064.96 a year."
+us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It counted a parent and two children as a household of four instead of three. It also used a $2,000 asset limit, but Montana TANF's resource limit is $3,000, so the $2,800 bank balance qualifies. Countable earnings of $219.69 a month are far below the payment standard."
+us,scenario_100,tanf,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It returned $0 because the facts did not show TANF receipt, but the prompt says to assume program take-up. TANF should have been computed from eligibility: about $725 minus $219.69 in countable earnings each month, or $6,064.96 a year."
+us,scenario_100,tanf,claude-opus-4.7,llm_error,categorical_eligibility,False,"It defaulted to $0 by vaguely pointing to asset, work, and income tests without applying any of them. Assets are under Montana's $3,000 limit with the vehicle excluded, 55 hours a week meets the work rules, and countable earnings of $219.69 leave a benefit of about $505 a month."
+us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"It claimed a $595 monthly benefit, which is above the $588 maximum it cited itself, so it never subtracted countable earnings. The correct benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or about $505.41 a month ($6,064.96 a year)."
+us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It assumed PolicyEngine gives this family no Montana TANF. The family meets the resource test and has countable earnings of only $219.69 a month after the $200 + 25% disregard, which leaves about $505 a month under the about $725 payment standard."
+us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"Its eligibility logic and the $200 + 25% earnings disregard ($220 a month countable) were correct, but it guessed a $730 payment standard. The 2026 three-person standard averages about $725.10 a month, so the benefit is $505.41 a month, not $510."
+us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It used a $1,000 liquid-asset limit, but Montana TANF's resource limit is $3,000, so $2,800 in bank assets passes. The family is eligible and gets about $505 a month after the earnings disregard."
+us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327 monthly payment standard, which is far below Montana's 2026 three-person standard of about $725. It also paid that near-maximum amount without subtracting the $219.69 in countable earnings left after the $200 + 25% disregard."
+us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The program is modeled, and this family qualifies for about $725 minus $219.69 in countable earnings each month, or $6,064.96 a year."
+us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to bank assets in the resource test. Montana TANF excludes the household vehicle, so countable resources are $2,800, under the $3,000 limit, and the family is eligible."
+us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It disregarded 50% of earnings above $200, but Montana disregards only 25%, so countable earnings are $219.69, not $146.46. It also used a $605 payment standard instead of the 2026 three-person standard of about $725.10."
+us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its reasoning computed a positive benefit of $4,023.75 but it submitted $0. That calculation also used a $555 payment standard instead of about $725.10, which understates the correct benefit of about $505 a month."
+us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 payment standard and a 50% disregard of earnings above $200. Montana disregards only 25%, so countable earnings are $219.69, and the 2026 three-person standard is about $725.10, which gives about $505 a month."
+us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the $200 + 25% disregard correctly ($220 countable) but used an outdated $588 payment standard. The 2026 three-person standard is about $725.10, so the benefit is about $505 a month, not $368."
+us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It claimed the family fails Montana's TANF requirements without naming any test. The family passes the $3,000 resource test, and its $219.69 in countable earnings is well below the about $725 payment standard, so it is owed about $505 a month."
+us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,asset_resource,False,"It assumed the head receives SSI, but the head's $2,800 in resources is over the $2,000 SSI limit for an individual, so there is no SSI income to offset TANF. With wages as the only income, TANF is about $505 a month."
+us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $5,298 total implies a 50% disregard of earnings above $200 and an outdated $588 payment standard. Montana disregards only 25%, so countable earnings are $219.69, and the 2026 standard is about $725.10."
+us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave $0 with no supporting derivation. The family is eligible, and the benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or $6,064.96 a year."
+us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"$5,298.48 equals ($588 − $146.46) × 12, so it used an outdated $588 payment standard and a 50% disregard. Montana's disregard is $200 plus 25% (countable $219.69), and the 2026 three-person standard is about $725.10."
+us,scenario_100,tanf,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Its explanation says the family qualifies and states a value of 4,824, yet it submitted $0. Even the $4,824 figure ($402 a month) understates the correct benefit of about $725.10 minus $219.69, or about $505 a month."
+us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It wrongly said Montana TANF is not modeled in PolicyEngine. The program is modeled, and it pays this family $6,064.96 a year based on the three-person payment standard minus disregarded earnings."
+us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium on top of $5,915 in wages, even though gross wages already include overtime. It also compared gross income to the payment standard without the $200 + 25% disregard. Countable earnings are only $219.69 a month, assets pass with the vehicle excluded, and 55 hours a week meets the work rules."
+us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,"It assumed Montana TANF gives this household nothing. The family is eligible, and the about $725.10 payment standard minus $219.69 in countable earnings gives $6,064.96 a year."
+us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It required an explicit indication of TANF payability, ignoring the instruction to assume take-up. A low-earning single parent with two children and assets under $3,000 qualifies for about $505 a month."
+us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It missed that a parent with two dependent children, low earnings, and resources under Montana's $3,000 limit meets TANF eligibility. The benefit is about $725.10 minus $219.69 each month, or $6,064.96 a year."
+us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"Its $150 countable income and $438 grant imply a 50% disregard and a $588 payment standard. Montana counts 75% of earnings above $200 ($219.69), and the 2026 three-person standard is about $725.10, so the benefit is about $505 a month."
+us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50 monthly benefit implies a payment standard below $590. The 2026 Montana three-person standard is about $725.10, and subtracting $219.69 in countable earnings gives about $505 a month."
+us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"$4,419.75 is ($588 − $219.69) × 12, so it applied the disregard correctly but used an outdated $588 payment standard. The 2026 three-person standard PolicyEngine applies is about $725.10."
+us,scenario_100,tanf,gpt-5.6-terra,llm_error,asset_resource,False,"It counted SSI income the head does not receive: the head's $2,800 in resources is over the $2,000 SSI limit, so SSI is $0. Countable earnings of $219.69 a month alone leave a TANF benefit of about $505 a month."
+us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"Its method was correct ($200 + 25% disregard, $2,636.25 a year in countable earnings), but it used a rounded $725 monthly payment standard. The 2026 standard PolicyEngine applies averages about $725.10 a month, which leaves its total $1.21 short of $6,064.96."
+us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"$5,298 matches a $588 payment standard with a 50% disregard of earnings above $200. Montana counts 75% of the excess ($219.69 a month), and the 2026 three-person standard is about $725.10."
+us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25 monthly benefit comes from an outdated $588 payment standard minus $219.69 in countable earnings. The 2026 three-person standard is about $725.10, which gives about $505 a month."
+us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,It applied the $200 + 25% disregard and the resource test correctly but used a $588 monthly payment standard instead of the 2026 three-person standard of about $725.10. That understates the benefit by about $137 a month.
+us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It did not compute TANF at all. The family is eligible, and the benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or $6,064.96 a year."
+us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"Its disregard ($220 countable) and asset findings were correct, but it used an outdated $588 payment standard. The 2026 three-person standard is about $725.10, which gives about $505 a month."
+us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It subtracted $220 in countable earnings from an outdated $588 payment standard. The 2026 Montana three-person standard is about $725.10, so the monthly benefit is about $505, not $368."
+us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"Every step was correct except the payment standard: it used $588 instead of the 2026 three-person standard of about $725.10. That understates the annual benefit by about $1,645."
+us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the disregard correctly but used a $675 payment standard. The 2026 three-person standard is about $725.10, so the benefit is about $505 a month, not $455."
+us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It took the $588 maximum grant as the 2026 payment standard. PolicyEngine's 2026 three-person standard is about $725.10, and subtracting $220 in countable earnings gives about $505 a month."
+us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no TANF value and no explanation, so there was no answer to score against $6,064.96."
+us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"Its resource test and $219.69 countable-earnings calculation were correct, but it used an outdated $588 three-person grant instead of the 2026 payment standard of about $725.10."
+us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana benefits are only $50–$200 a month and that earnings reduce them to zero. The three-person payment standard is about $725.10, and countable earnings after the $200 + 25% disregard are just $219.69, which leaves about $505 a month."
+us,scenario_100,tanf,ox-alpha,llm_error,asset_resource,False,"It counted about $780 a month in SSI as unearned income, but the head is not SSI-eligible because $2,800 in resources is over the $2,000 SSI limit. With only $219.69 in countable earnings, TANF is about $505 a month."
+us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium on top of $5,915 in wages, even though gross wages already include overtime. It then compared gross income to an annual limit without the $200 + 25% earnings disregard, which brings countable earnings down to $219.69 a month."
+us,scenario_100,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"It said income was too high without applying Montana's $200 + 25% earnings disregard. Countable earnings are $219.69 a month, well below the about $725.10 payment standard, so the benefit is $6,064.96 a year."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model subtracted $1,400 of auto loan interest on top of the $16,100 standard deduction, which lowered taxable income to $83,630. The facts never show a new, U.S.-assembled vehicle bought with a post-2024 loan, so no qualified passenger vehicle loan interest deduction applies. Its final $12,064 does not even match its own $13,058–$13,111 working, and the correct figure is $13,418.60 on $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model's standard deduction and 2026 brackets were right, but it also subtracted a $1,000 phased-down auto loan interest deduction. The listed loan is not shown to be a qualified passenger vehicle loan, so that deduction does not apply. That left taxable income at $84,030 instead of $85,030 and understated tax by $220."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model counted the nontaxable $700 of veterans benefits in AGI. It used the 2024 $14,600 standard deduction and $11,600 bracket edge instead of the 2026 figures ($16,100 and $12,400/$50,400), and it taxed nearly all income at 12% while skipping the 22% bracket. Its stated $10,637.50 does not match its submitted $11,435."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model subtracted the health premiums from wages as if they were pre-tax, but no pre-tax exclusion is listed. It showed no bracket arithmetic. Its $13,900 implies about $87,200 of taxable income under 2026 brackets, roughly $2,200 above the correct $85,030, which fits an old standard deduction of about $15,000 rather than $16,100."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed exactly $13,418.60 from $85,030 of taxable income and the $12,400/$50,400 bracket edges. It then threw that result away and submitted $14,716 under the label 'rounding to bracket estimates', a number with no basis in its own working."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model derived taxable income of $85,030 and tax of about $13,419 using the correct 2026 brackets. It then replaced that with an unsupported $13,876, which none of its stated parameters produce."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model took the full $1,400 of auto loan interest as a qualified vehicle loan interest deduction, dropping taxable income to $83,630. Nothing establishes a new, U.S.-assembled vehicle with a post-2024 loan, so the only deduction is the $16,100 standard deduction. Tax should be on $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 indexed $16,100. It also used guessed bracket edges of $12,300/$49,950 instead of $12,400/$50,400. This overstated taxable income by $350 and misplaced the bracket boundaries, giving $13,543."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model got taxable income right at $85,030 but used wrong 2026 bracket edges ($12,150/$49,400 instead of $12,400/$50,400), which gave $13,523.60. It then submitted an unexplained $13,322 that matches neither its own working nor the correct $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,174 auto loan interest deduction, lowering taxable income to $83,856. The loan is not shown to be a qualified passenger vehicle loan (new vehicle, U.S. final assembly, originated after 2024), so no such deduction applies. Taxable income stays at $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction instead of the 2026 $16,100, and bracket edges of $12,175/$49,550 instead of $12,400/$50,400. Taxable income came out at $85,530, and pushing more income into the 22% bracket inflated tax to $13,618."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted $3,840 of 'pre-tax' health premiums from wages, counting the same $1,920 premium twice, even though no salary-reduction exclusion is listed. It also used a $15,400 standard deduction and $12,200/$49,700 bracket edges instead of $16,100 and $12,400/$50,400."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset rules: an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA rate structure permanent and set the 2026 single standard deduction at $16,100 with no personal exemption, so taxable income is $85,030, taxed at 10/12/22%."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,350 standard deduction instead of $16,100, and bracket edges of $12,200/$49,575 instead of $12,400/$50,400. Taxable income came out at $85,780 and tax at $13,670.10 instead of $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired, using an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. Under OBBBA, 2026 keeps 10/12/22% brackets with a $16,100 standard deduction and no exemption. It also subtracted $1,920 of premiums as pre-tax with no listed exclusion."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model started from $103,000 of wages and ignored the $1,870 capital loss that reduces AGI to $101,130. It used a $15,300 standard deduction instead of $16,100, ending with $87,700 of taxable income instead of $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model got AGI right at $101,130 but showed no deduction or bracket calculation. Its $15,400 is about $1,980 too high: subtracting the $16,100 standard deduction and applying 10% to $12,400, 12% to $50,400 and 22% above gives $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated 2026 as a post-TCJA-expiration year, using an $8,250 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA brackets permanent, and the 2026 single standard deduction is $16,100 with no exemption."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used a standard deduction of about $15,000 and never worked through the brackets. Its $11,102 implies only about $74,500 of taxable income under 2026 brackets, roughly $10,500 short of the correct $85,030 (AGI $101,130 minus the $16,100 standard deduction)."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset rules (an $8,300 standard deduction plus a $5,300 personal exemption, with 15%/25% brackets) and also used outdated $11,000/$44,700 bracket edges. Under current law, 2026 uses 10/12/22% brackets with edges at $12,400/$50,400 and a $16,100 standard deduction."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted an $8,300 standard deduction and a $5,300 personal exemption on the assumption that the TCJA sunset in 2026. OBBBA kept the TCJA structure, so the correct deduction is $16,100 with no exemption, and $85,030 is taxed at 10/12/22%."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket edges of $11,925/$48,475. The 2026 figures are a $16,100 deduction and $12,400/$50,400 edges, which give $85,030 of taxable income and $13,418.60 of tax."
+us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model extrapolated its own 2026 parameters: a $15,450 standard deduction and bracket edges of $12,283/$49,929. The published 2026 values are $16,100 and $12,400/$50,400, so its taxable income was $650 too high and its brackets were in the wrong places."
+us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model's 2026 brackets were right, but it used a $15,800 standard deduction instead of the 2026 indexed $16,100. That left taxable income at $85,330 instead of $85,030 and overstated tax by $66."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model added 'taxable pension income' and 'small taxable capital gains' to wages. The $2,308 pension is tax-exempt, and the $1,870 net capital loss reduces AGI to $101,130 rather than adding gain. That inflated taxable income above $85,030 and tax above $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model claimed itemized medical effects, but $2,520 of medical costs is far below the 7.5%-of-AGI floor ($7,585), and the standard deduction wins anyway. It showed no bracket calculation. Its $9,587 is about $3,800 below the $13,418.60 that results from $85,030 of taxable income under 2026 brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model assumed the scheduled TCJA sunset and used a pre-TCJA standard deduction plus a personal exemption, leaving about $87,380 of taxable income taxed at the old higher rates. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026 with no exemption."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model subtracted a phased-down auto loan interest deduction on top of the standard deduction. The listed loan is never shown to be a qualified passenger vehicle loan (new, U.S.-assembled vehicle, loan originated after 2024), so that deduction is zero. Taxable income stays at $85,030 rather than $84,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,000 auto loan interest deduction to reach $84,030 of taxable income. No qualified passenger vehicle loan is established, so the only deduction is the $16,100 standard deduction, giving $85,030 and $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction instead of $16,100, which overstated taxable income at $85,830. Its $13,774 also exceeds what the correct 2026 brackets produce on that income ($13,594.60), so it used bracket edges below $12,400/$50,400 as well."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset structure: about $8,400 of standard deduction plus a $5,300 exemption, taxed at 10/15/25%. Current law for 2026 keeps 10/12/22% brackets and a $16,100 standard deduction with no exemption."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket edges of $11,925/$48,475 instead of the 2026 values ($16,100 and $12,400/$50,400). That overstated taxable income by $1,100 and misplaced the bracket boundaries."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got taxable income right at $85,030 but used bracket edges of $12,200/$49,575 instead of the 2026 values of $12,400/$50,400. That pushed $825 too much income into the 22% bracket and $200 too much into the 12% bracket, overstating tax by $86.50."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA reverted in 2026, using an $8,300 standard deduction, a $5,300 exemption, and 15%/25% brackets. OBBBA made the TCJA rates permanent, with a $16,100 standard deduction and no exemption for 2026."
+us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly reached $85,030 of taxable income, but it reported an approximate tax of $13,426 instead of summing the brackets exactly. The exact sum is $1,240 + $4,560 + 22% × $34,630 = $13,418.60, so its figure is $7.40 high."
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so no answer could be scored."
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,174 auto loan interest deduction, cutting taxable income to $83,856. The loan is not established as a qualified passenger vehicle loan, so no such deduction applies. Taxable income stays at $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model's own reasoning acknowledged that the standard deduction and 2026 brackets leave a positive tax, but it submitted $0 anyway. With no credits to offset it, tax on $85,030 of taxable income is $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction instead of $16,100 and bracket edges of $12,150/$49,000 instead of $12,400/$50,400, which gave $13,641. It then submitted $14,827, which contradicts its own calculation."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model netted the $1,870 capital loss against tax-exempt pension income instead of deducting it from AGI, and it used a pre-TCJA-style $13,600 deduction instead of $16,100. It taxed everything above $12,400 at 12%, skipping the 22% bracket. It then subtracted employee payroll tax as an invented nonrefundable credit."
us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,386 of Social Security tax and $1,493.50 of Medicare tax, then submitted $8,149.50 despite its own arithmetic totaling $7,879.50. Its unexplained $270 addition is not produced by a wage-base cap, rounding, Additional Medicare Tax, or Texas employee payroll tax."
-us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated total employee payroll tax as $7,879.50 but submitted $7,854.00. The $25.50 reduction has no basis in any payroll-tax component stated in its reasoning."
-us,scenario_101,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly deducted the $1,920 health-insurance premium from FICA wages. The prompt identifies premiums paid but does not establish a pre-tax employer-plan salary reduction, so both Social Security and Medicare taxes apply to the full $103,000 of wages."
-us,scenario_101,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly computed $7,879.50, then replaced it with $7,757 using a nonexistent PolicyEngine rounding adjustment. Texas adds no mandatory employee payroll tax here, and rounding cannot reduce the correctly summed components by $122.50."
-us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract."
-us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated ordinary withholding of employee FICA taxes as though it eliminated annual payroll-tax liability. W-2 wages of $103,000 generate $6,386 of employee Social Security tax and $1,493.50 of employee Medicare tax regardless of employer-sponsored insurance or Texas's lack of a state payroll tax."
-us,scenario_101,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly summed Social Security and Medicare taxes to $7,879.50, then added an unexplained $152. No Additional Medicare Tax or mandatory Texas employee payroll tax applies, so that extra amount has no payroll-tax source."
+us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It correctly computed $6,386 Social Security plus $1,493.50 Medicare = $7,879.50. It then submitted $8,149.50, citing a made-up 'adjustment for wage base cap and rounding.' The cap does not bind at $103,000 of wages, so no adjustment applies and the submitted value contradicts its own arithmetic."
+us,scenario_101,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its components ($6,386 Social Security plus about $1,494 Medicare) add to $7,879.50, but it submitted $8,803. That figure does not follow from any of the components it listed, so this is an addition error. No extra payroll tax applies in Texas."
+us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"It explicitly reached a total of $7,879.50 with the correct 6.2% and 1.45% rates on $103,000. It then submitted $7,854 anyway, a $25.50 transcription error that contradicts its own stated total."
+us,scenario_101,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It cut the FICA wage base to $101,080 by subtracting the $1,920 of health insurance premiums as if they were a pre-tax Section 125 deduction. The listed premiums are not a reduction to gross wages, so FICA applies to the full $103,000 of wages, giving $7,879.50."
+us,scenario_101,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It computed the correct $7,879.50 total and then lowered it to $7,757, attributing the change to an invented 'PolicyEngine rounding.' No rounding or offset applies, so the submitted value contradicts its own correct arithmetic."
+us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is 6.2% × $103,000 + 1.45% × $103,000 = $7,879.50."
+us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It acknowledged that $103,000 of W-2 wages are subject to Social Security and Medicare withholding, then submitted $0 anyway. Its reason was that Texas has no state payroll tax, which confuses state and federal FICA. Employee FICA on these wages is $7,879.50 whatever the state."
+us,scenario_101,payroll_tax,qwen3.8-max,llm_error,other,False,"It correctly totaled $6,386 plus $1,493.50 = $7,879.50. It then added an unexplained $152 to reach $8,031.50. No Additional Medicare Tax or state employee payroll tax applies, so the extra $152 has no basis."
us,scenario_101,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invented a Texas individual income tax liability by applying an unstated tax rate to taxable income. Texas imposes no individual state income tax, so the correct state income tax before refundable credits is $0, not $6,035."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed about $6,092 using a $32,300 standard deduction ($100 above the actual $32,200, with no QBI deduction), and about $5,133 after wrongly excluding the ESI premium. It then submitted $5,459, which matches neither calculation. The correct figure on $55,000.80 of taxable income is $6,104.10."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It used the 2024 standard deduction of $29,200 instead of 2026's $32,200. It then subtracted an invented $2,810 of 'withholding' from the tax. Withholding is a payment, not a nonrefundable credit, so it never reduces income tax before refundable credits."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $31,500 and a $24,000 10% bracket instead of 2026's $32,200 and $24,800, which gave $6,204. It then submitted $5,764, which does not follow from its own arithmetic."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It worked through the brackets to about $6,092, using a $32,300 standard deduction instead of $32,200 and no QBI deduction. It then submitted $6,633, which contradicts its own computation; the correct figure is $6,104.10."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It followed the correct derivation ($32,200 standard deduction, about $55,001 taxable income, $2,480 + $3,624 = $6,104) and then submitted a 'rounded estimate' of $5,983. That unexplained $121 reduction has no basis in any credit or deduction."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 figures ($30,000 standard deduction, $23,850 10% bracket) by 2.7% to $30,800 and $24,500. It missed OBBBA raising the 2026 amounts to $32,200 and $24,800, which overstated taxable income by $1,400."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a $32,600 standard deduction instead of $32,200 and reached $6,056. It then added an invented 'SE tax adjustment' to get $6,800. Self-employment tax is not part of income tax, and $1 of self-employment income is below the $400 threshold, so there is no SE tax at all."
-us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction and a $24,000 10% bracket instead of 2026's OBBBA amounts of $32,200 and $24,800. That overstated taxable income by $1,600 and gave $6,312."
-us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $30,000 and the 2025 10% bracket top of $23,850, not 2026's $32,200 and $24,800. That overstated taxable income by $2,200 and tax by $283."
-us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $16,600 standard deduction, $10,600 of personal exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, no personal exemptions, and 10%/12% brackets."
-us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,500 10% bracket, which are inflation-projected pre-OBBBA values. The 2026 amounts are $32,200 and $24,800, so it overstated taxable income by $1,400."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset ($16,000 standard deduction, $10,300 of exemptions, a 15% bracket) even though OBBBA made the $32,200 standard deduction and 10%/12% brackets permanent. It also subtracted the $7,989 ESI premium from AGI even though the stated gross wages are fully taxable."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the pre-OBBBA $30,000 standard deduction instead of $32,200. It then reported $4,124 on $57,201 of taxable income, which is far below what even its own inputs give under the 10%/12% brackets (about $6,387)."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the pre-TCJA 2026 structure ($15,850 standard deduction plus $10,100 of personal exemptions, 15% bracket) that OBBBA repealed. It also subtracted the $7,989 ESI premium from AGI, which wrongly reduced taxable wages."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used post-sunset brackets (10% to $22,000, then 15%) and a $16,000 standard deduction. OBBBA's permanent 2026 schedule is a $32,200 standard deduction with 10% to $24,800 and 12% above. It also wrongly subtracted the $7,989 ESI premium from AGI."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It reported $0 with no derivation. After the $32,200 standard deduction, the couple has $55,000.80 of taxable income and no nonrefundable credits to offset the resulting $6,104.10 of tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium as a pre-tax exclusion, lowering AGI to $79,212, although the stated gross wages are fully taxable. It also used a $30,800 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions and the post-sunset rate schedule to 2026. OBBBA made the TCJA's zero-exemption, $32,200 standard deduction and 10%/12% bracket structure permanent, so its $7,915.15 overstates tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $30,000 and the 2025 10% bracket top of $23,850 instead of 2026's $32,200 and $24,800. That overstated tax at $6,387.12."
-us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It computed $6,398.12 using the outdated $30,000 standard deduction and a $23,800 10% bracket. It then submitted $1,737, which contradicts its own calculation and is backed by no credit."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It said no nonrefundable credits apply, yet reported $3,357, roughly half the correct liability. Applying the $32,200 standard deduction and the 10%/12% brackets to $87,201 of income gives $6,104.10, so its figure understates taxable income or bracket tax with no supporting rule."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It reported zero tax on $87,200 of wages while listing no credits. The $32,200 standard deduction shelters only part of the income and leaves $55,000.80 taxable, which produces $6,104.10 of tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $7,989 ESI premium as a pre-tax salary reduction and cut AGI to $79,212. The prompt lists gross wages and no cafeteria-plan election, so wages stay fully taxable. That exclusion alone lowered tax by $958.68 to $5,145.44."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It excluded the $7,989 ESI premium from wages as if it were a pre-tax cafeteria-plan deduction, although the stated gross wages are fully taxable. It also claimed a half-SE-tax deduction even though $1 of self-employment income is below the $400 threshold."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It got the $32,200 standard deduction and $0.20 QBI deduction right, but subtracted the $7,989 ESI premium as a pretax employer-plan exclusion and reduced AGI to $79,212. The stated gross wages are fully taxable, so taxable income is $55,000.80 and tax is $6,104.10."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $57,200 implies a pre-OBBBA $30,000 standard deduction instead of 2026's $32,200. The $6,200 is a rounded figure not tied to the 2026 brackets (10% to $24,800, 12% above)."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected the pre-OBBBA 2025 values forward to a $30,840 standard deduction and a $24,518 10% bracket. The actual 2026 figures are $32,200 and $24,800, so it overstated taxable income by about $1,360."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired, with a $16,550 standard deduction, $10,600 of personal exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction and 10%/12% brackets with no exemptions."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,500 10% bracket, which are projected pre-OBBBA values. OBBBA set 2026 at $32,200 and $24,800, so it overstated taxable income by $1,400 and tax by $174."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium from wages to reach a $79,212 AGI, although the stated gross wages are fully taxable. It also used a $30,800 standard deduction instead of 2026's $32,200."
-us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so nothing substantive can be scored."
-us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used an estimated $30,750 standard deduction and a $24,450 10% bracket instead of OBBBA's 2026 amounts of $32,200 and $24,800. That overstated taxable income by $1,450."
-us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed a Child and Dependent Care Credit, but the household has no qualifying dependent and no care expenses, so the CDCC is zero. It also used an inflated $33,500 standard deduction instead of $32,200. Its final $3,602 does not even equal its own $6,324 minus $600."
-us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It invented two qualifying children and subtracted a $4,000 Child Tax Credit, but the household is a childless couple with no CTC. It also used the outdated $30,000 standard deduction and overstated pre-credit tax at $8,631."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used a $32,300 standard deduction instead of the 2026 OBBBA figure of $32,200, and its own bracket math gave about $6,092. It then submitted $5,459, which does not follow from either its no-exclusion figure ($6,092) or its ESI-exclusion figure ($5,133)."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 standard deduction of $29,200 instead of the 2026 figure of $32,200. It then subtracted an assumed $2,810 of wage withholding, but withholding is a payment, not a nonrefundable credit, and it never reduces tax before refundable credits."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It reached the correct taxable income of about $55,000 but misstated the tax as about $5,600; the correct figure is $2,480 + 12% × $30,200.80 = $6,104.10. It then cut the answer to $1,900 for a made-up nonrefundable credit, although a childless couple with no care expenses has none."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $31,500 and a $24,000 top for the 10% bracket instead of the 2026 values of $32,200 and $24,800, which gave it $6,204. It then submitted an unexplained $5,764."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $32,300 standard deduction instead of $32,200 and computed about $6,092. It then submitted $6,633, which does not match its own bracket calculation."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly found taxable income of about $55,001 with the $24,800 10% bracket and computed about $6,104. It then dropped that result and submitted an unexplained 'rounded estimate' of $5,983."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 values ($30,000 standard deduction and $23,850 bracket) to $30,800 and $24,500. It missed that OBBBA raised the 2026 MFJ standard deduction to $32,200, with the 10% bracket ending at $24,800, and so overstated taxable income by $1,400."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $32,600 standard deduction instead of $32,200 and computed $6,056. It then raised that to $6,800 for an 'SE tax adjustment', but $1 of self-employment income creates no SE tax, and SE tax is not part of income tax in any case."
+us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction and a $24,000 top for the 10% bracket instead of the OBBBA 2026 values of $32,200 and $24,800. That pushed taxable income up to $56,601 and the tax up to $6,312."
+us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 parameters ($30,000 standard deduction and a $23,850 10% bracket) to tax year 2026. The 2026 values are a $32,200 deduction and a $24,800 bracket, so it overstated taxable income at $57,201."
+us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a $16,600 standard deduction, $10,600 of personal exemptions and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, no exemptions, and 10%/12% brackets."
+us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used inflation-only projections of the pre-OBBBA values ($30,800 standard deduction and a $24,500 10% bracket) instead of the 2026 OBBBA values of $32,200 and $24,800. That gave taxable income of $56,401 instead of $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and applied a $16,000 standard deduction, $10,300 of exemptions and a 15% bracket, though OBBBA made the $32,200 deduction and 12% bracket permanent. It also wrongly cut AGI to $79,212 by subtracting the $7,989 ESI premium from the stated gross wages."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the pre-OBBBA $30,000 standard deduction instead of $32,200, giving taxable income of $57,201. Its submitted $4,124 is also far below what the 10%/12% brackets produce even on that income (about $6,387)."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and applied a $15,850 standard deduction plus $10,100 of exemptions with pre-TCJA brackets, though OBBBA made the $32,200 deduction and 10%/12% brackets permanent. It also wrongly subtracted the $7,989 ESI premium from the stated gross wages."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied post-TCJA-expiration rules (a $16,000 standard deduction and a 15% bracket above $22,000) instead of OBBBA's permanent $32,200 deduction and 12% bracket. It also wrongly reduced AGI by the $7,989 ESI premium, though the stated wages are the gross wages used in the calculation."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation and reported $0. The couple has $55,000.80 of taxable income after the $32,200 standard deduction and no nonrefundable credits, so the tax is $6,104.10. A zero answer ignores the positive regular tax entirely."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium from the stated gross wages, giving AGI of $79,212 instead of $87,201. It also used the pre-OBBBA $30,800 standard deduction and $23,850 bracket instead of $32,200 and $24,800."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied 2026 rules as if the TCJA had expired, with personal exemptions and the pre-TCJA standard deduction. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 parameters ($30,000 standard deduction and a $23,850 10% bracket) for 2026. The OBBBA 2026 values are $32,200 and $24,800, so it overstated taxable income at $57,201."
+us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used the pre-OBBBA $30,000 standard deduction and computed $6,398.12, then said no credits apply. It still submitted $1,737, which contradicts its own calculation."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no computation and reported $3,357, about half the correct liability. The correct path is $87,201 of income minus the $32,200 standard deduction and $0.20 QBI deduction, giving taxable income of $55,000.80 and tax of $6,104.10 with no credits."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It claimed zero tax after the standard deduction and credits, but $87,201 of income far exceeds the $32,200 standard deduction. No nonrefundable credit applies to the $6,104.10 of regular tax on $55,000.80 of taxable income."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 employer-sponsored premium from the stated gross wages as a pre-tax exclusion, which dropped taxable income to $47,011.80. The ESI premium is not netted out of the listed wages, so taxable income is $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated the $7,989 employer-sponsored premium as a pre-tax reduction of the listed gross wages, which cut taxable income by $7,989 and gave $5,145.42. The wages enter AGI in full, which gives $87,201 of income and $55,000.80 of taxable income."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It correctly used the $32,200 standard deduction and $0.20 QBI deduction, but it subtracted the $7,989 ESI premium from wages to reach AGI of $79,212. The stated gross wages enter AGI in full at $87,201, and that one wrong exclusion explains the gap."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $57,200 means it used the pre-OBBBA $30,000 standard deduction instead of the 2026 figure of $32,200. That overstated taxable income by about $2,200."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It inflation-projected the pre-OBBBA 2025 values to a $30,840 standard deduction and a $24,518 10% bracket. It missed OBBBA's 2026 values of $32,200 and $24,800."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a $16,550 standard deduction, $10,600 of exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 deduction with no exemptions and a 12% second bracket."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly kept the TCJA rate structure but used inflation-only projections ($30,800 standard deduction and a $24,500 10% bracket). It missed OBBBA's increase to a $32,200 MFJ standard deduction with the 10% bracket ending at $24,800."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium from the stated gross wages to get AGI of $79,212 instead of $87,201. It also used the pre-OBBBA $30,800 standard deduction instead of $32,200."
+us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $6,104.10 liability."
+us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used projected pre-OBBBA values ($30,750 standard deduction and a $24,450 10% bracket) instead of the 2026 OBBBA values of $32,200 and $24,800. That gave taxable income of $56,451 instead of $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It used a nonexistent $33,500 standard deduction and subtracted a Child and Dependent Care Credit, though the household reports no dependent-care expenses, so the CDCC is $0. Its final $3,602 also does not equal its own $6,324 minus the $600 credit it claimed."
+us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It made up two qualifying children and subtracted a $4,000 Child Tax Credit, but the household is a childless couple with no CTC eligibility. It also used the pre-OBBBA $30,000 standard deduction and miscomputed the bracket tax as $8,631."
us,scenario_102,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a $3,995 refundable disability-related credit even after correctly ruling out EITC, refundable CTC, and the American Opportunity Credit. The Credit for the Elderly or the Disabled is nonrefundable, disability alone does not establish it for these working 26-year-olds, and no listed fact generates any refundable component."
us,scenario_102,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract."
us,scenario_102,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented a medically needy spend-down pathway in the PolicyEngine determination and improperly treated rent as a Medicaid spend-down expense. The head qualifies for no Medicaid category, and the household's $87,201 MAGI exceeds every applicable income threshold, so the listed expenses and assets do not change the result."
@@ -8251,23 +8458,24 @@ us,scenario_102,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,Fals
us,scenario_102,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model's disability-based Medicare rule was triggered by the disability flag alone. That skips the 24-month SSDI entitlement requirement. The head has no SSDI and earns $41,200 in wages, so the disability pathway is closed, and at 26 the head also fails the age-65 test."
us,scenario_102,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model's modeled disability pathway was triggered just by the head's disability flag. That pathway only applies after 24 months of SSDI entitlement. The head has zero SSDI and $41,200 in wages, so the only test left is age 65 or older, which a 26-year-old fails."
us,scenario_102,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as 'at least 65 or disabled', but PolicyEngine's actual rule is age 65 or older, or 24 months of SSDI receipt (or ESRD/ALS). The head has no SSDI, and $41,200 in wages is far above substantial gainful activity. The disability flag alone does not qualify a 26-year-old, so the answer is 0."
-us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used 14.5% instead of the 1.45% employee Medicare rate, which gave $12,644 of Medicare tax. It then switched between subtracting the $8,389 of premiums and not subtracting them, and submitted $7,149.68, a figure that none of its own calculations produce. The correct figure is 7.65% of the full $87,200 of gross wages, which is $6,670.80."
-us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"It correctly calculated $3,151.80 + $3,519.00 = $6,670.80 at 7.65% of each spouse's gross wages. It then submitted $6,672.30, a transcription error that doesn't match its own total."
-us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"It correctly found Social Security tax of $5,406.40 and Medicare tax of $1,264.40, which add to $6,670.80. It then rounded to 'about $6,671' and submitted $6,672, which is neither the exact sum nor a correct rounding of it."
-us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"It correctly broke out both spouses' taxes ($2,554.40 + $597.40 + $2,852.00 + $667.00) and stated the total as $6,670.80. It then submitted $6,664.60, a transcription error that contradicts its own sum."
-us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the spouse's $7,989 employer-sponsored insurance premium as a pre-tax salary reduction and used $79,211 as the FICA base. FICA applies to the full listed gross wages of $87,200, which gives $6,670.80."
-us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut the spouse's FICA wages to $38,011 by subtracting the $7,989 employer-sponsored insurance premium. The spouse's full $46,000 of gross wages is subject to FICA, which gives $3,519.00 for the spouse and $6,670.80 for the household."
-us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $7,989 employer-sponsored insurance premium as pre-tax and used $79,211 of FICA-taxable wages. The listed gross wages of $41,200 and $46,000 are the payroll tax base with no premium exclusion, so 7.65% × $87,200 = $6,670.80."
-us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $7,989 employer-sponsored insurance premium from the spouse's wages to get a $38,011 FICA base. The employee Social Security and Medicare taxes apply to the spouse's full $46,000, which gives $6,670.80 for the household."
-us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"It correctly calculated $5,406.40 + $1,264.40 = $6,670.80 on $87,200 of wages. It then submitted $6,667.80, a digit transcription error."
-us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It used the correct base and rate, (41,200 + 46,000) × 0.0765, but got the multiplication wrong and reported $7,063. The product is $6,670.80."
-us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It calculated FICA on $79,211 after subtracting the spouse's $7,989 employer-sponsored premium as if it were pre-tax. FICA applies to the full $87,200 of gross wages, which gives $6,670.80."
-us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It reduced the spouse's wages by the $7,989 employer-sponsored premium, treating it as a pre-tax salary reduction. Employee FICA applies to the spouse's full $46,000 of gross wages, which gives $6,670.80 for the household."
-us,scenario_102,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It applied 7.65% to $79,211 after subtracting the $7,989 employer-plan premium as a pre-tax deduction. The payroll tax base is the full $87,200 of listed gross wages, which gives $6,670.80."
-us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $7,989 employer-sponsored insurance premium from the spouse's wages, used a $79,211 FICA base, and rounded the result to $6,060. The spouse's full $46,000 is FICA wages, so the household total is 7.65% × $87,200 = $6,670.80."
-us,scenario_102,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score. The correct result is 7.65% of the $87,200 of combined gross wages, which is $6,670.80."
-us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly found Social Security tax of $5,406.40 and Medicare tax of $1,264.40, a total of about $6,671. It then submitted $6,539, a number its own arithmetic does not support."
-us,scenario_102,payroll_tax,qwen3.8-max,llm_error,other,False,"It used $87,000 of total wages instead of $41,200 + $46,000 = $87,200. It then added its own $5,394 and $1,262 wrongly and submitted $6,681 instead of their $6,656 sum. The correct total on $87,200 is $6,670.80."
+us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used 14.5% instead of 1.45% for employee Medicare tax, which produced $12,644 of Medicare tax. It then switched back and forth on subtracting the ESI and health premiums from wages, and its submitted $7,149.68 does not follow from any of its own calculations. The correct calculation is 7.65% of $87,200 in gross wages, which is $6,670.80."
+us,scenario_102,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It used the correct 6.2% + 1.45% rates on $87,200 but rounded the components to $5,406 and $1,264. It then added them as $6,669 instead of $6,670, and dropped the cents of the correct $6,670.80."
+us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning correctly computed 7.65% × $41,200 = $3,151.80 and 7.65% × $46,000 = $3,519.00, a total of $6,670.80. It then submitted $6,672.30, a transcription error that does not match its own total."
+us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"It correctly derived Social Security tax of $5,406.40 plus Medicare tax of $1,264.40, but rounded the total to $6,671 and then submitted $6,672. The submitted number does not match its own exact sum of $6,670.80."
+us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"It computed each spouse's Social Security and Medicare amounts correctly and wrote the total as $6,670.80, but submitted $6,664.60. The submitted value is a transcription error that contradicts its own stated sum."
+us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the spouse's $7,989 employer-sponsored insurance premium as a pre-tax reduction of FICA wages, which cut the base to $79,211. FICA applies to the full listed gross wages of $87,200, so the correct total is $6,670.80, not $6,059.64."
+us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It reduced the spouse's FICA wages from $46,000 to $38,011 by subtracting the $7,989 ESI premium, assuming a cafeteria-plan salary reduction that the facts never state. The spouse's tax is 7.65% × $46,000 = $3,519.00, which makes the household total $6,670.80."
+us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $7,989 ESI premium as a pre-tax deduction from FICA wages, which produced a $79,211 base. Employee Social Security and Medicare tax apply to the full $87,200 of gross wages, giving $6,670.80."
+us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $7,989 ESI premium from the spouse's wages before applying 7.65%. The spouse's FICA base is the full $46,000 of listed gross wages, so the correct household total is $6,670.80, not $6,059.64."
+us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"Its reasoning correctly summed $5,406.40 of Social Security tax and $1,264.40 of Medicare tax to get $6,670.80. It then submitted $6,667.80, a digit transcription error."
+us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It set up the correct formula, $87,200 × 0.0765, but evaluated it as $7,063, which matches an 8.1% rate. The correct product is $6,670.80."
+us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It computed FICA on $79,211 of wages after subtracting the spouse's $7,989 employer-sponsored premium. FICA applies to the full $87,200 of gross wages, giving $5,406.40 of Social Security tax plus $1,264.40 of Medicare tax, a total of $6,670.80."
+us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It reduced the spouse's FICA wages by treating the $7,989 ESI premium as a pre-tax deduction. Employee payroll tax applies to the full $46,000 and $41,200 of gross wages, a total of $6,670.80."
+us,scenario_102,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It applied 7.65% to $79,211 after removing pretax employer-plan premiums, but no salary-reduction election is given. The FICA base is the full $87,200 of listed gross wages, so the correct total is $6,670.80."
+us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $7,989 ESI premium from the spouse's wages, which gave a $79,211 base, and then rounded the result to $6,060. The FICA base is the full $87,200 of gross wages, so the correct total is $6,670.80."
+us,scenario_102,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so its answer was missing rather than wrong. The correct answer is 7.65% of $87,200 in gross wages, which is $6,670.80."
+us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $5,406.40 + $1,264.40 ≈ $6,671 and then claimed that rounding gives $6,539. That figure has no connection to its own calculation and is an arithmetic and transcription error."
+us,scenario_102,payroll_tax,qwen3.8-max,llm_error,other,False,"It misread total wages as $87,000 instead of $87,200, which gave component taxes of $5,394 and $1,262. It then added those as $6,681 instead of $6,656. On the correct $87,200 base, the total is $6,670.80."
us,scenario_102,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a $31 minimum or rounding treatment after acknowledging that $1 of self-employment income produces approximately zero tax. Net earnings at this level do not reach the self-employment-tax filing threshold, so the liability is $0."
us,scenario_102,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested self_employment_tax output entirely. The listed $1 of self-employment income is below the threshold for self-employment tax, yielding $0."
us,scenario_102,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model said PolicyEngine grants Medicare to any disabled person regardless of age. In fact, the disability pathway under 65 requires 24 months of SSDI entitlement (or ESRD/ALS), and the 26-year-old spouse receives no SSDI. With the age-65 test failed and no SSDI history, the spouse is not eligible, so the answer should have been 0."
@@ -8277,32 +8485,33 @@ us,scenario_102,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,Fa
us,scenario_102,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model used a disability-based Medicare rule keyed only to the disability flag. That pathway requires 24 months of SSDI entitlement, and the spouse receives no SSDI. At age 26 with no SSDI history and no ESRD/ALS, the spouse fails every Medicare pathway, so the answer should have been 0."
us,scenario_102,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model called on the 'modeled disability-based Medicare pathway' but treated the disability flag alone as satisfying it. That pathway turns on 24 months of SSDI receipt, and the spouse's SSDI is $0. The 26-year-old spouse is therefore not Medicare eligible."
us,scenario_102,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as 'age 65 or disabled,' which drops the requirement that the disability pathway runs through 24 months of SSDI entitlement (or ESRD/ALS). The spouse is 26 and receives no SSDI, so neither condition is met and eligibility is 0."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reached the correct $61,701 × 3.99% ≈ $2,462. It then wrongly claimed the 2026 rate of 3.99% depends on revenue triggers, switched to 4.25%, and applied that to an invented $64,830 base to get $2,755. The 3.99% rate is set by statute for 2026, and the base is $87,201 minus the $25,500 NC joint standard deduction."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed North Carolina has no state income tax. In fact NC levies a 3.99% flat individual income tax in 2026. Applied to $61,701 of NC taxable income ($87,201 AGI minus the $25,500 joint standard deduction), that gives $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the base right ($87,201 − $25,500 = $61,701) but used the 2025 rate of 4.25% instead of the 2026 statutory rate of 3.99%. It then submitted $2,587, which does not even match its own $2,622 result. The correct figure is $61,701 × 3.99% = $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning twice arrives at the correct $61,701 × 3.99% ≈ $2,462, but it submitted $3,389, which no step of its derivation produces. That number amounts to taxing about $79,700 at 4.25%, which drops most of the $25,500 joint standard deduction and uses the superseded 2025 rate."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It computed the correct ~$2,462 from 3.99% of $61,700, then replaced it with $2,792. That number equals $65,701 × 4.25%, which means it used the outdated $21,500 joint standard deduction and the 2025 rate instead of NC's $25,500 deduction and the 3.99% rate for 2026."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $21,500 as the NC married-filing-jointly standard deduction, but that deduction has been $25,500 since 2022. It also used a 4.25% rate instead of the 3.99% statutory rate for 2026, so it taxed $65,701 at 4.25% rather than $61,701 at 3.99%."
-us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the correct $61,701 taxable income but applied the 2025 rate of 4.25% (giving $2,622.29) instead of 3.99%. It then inflated the result to $3,846 with an unexplained 'adjustment' that has no basis in NC law."
-us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It subtracted a $30,000 standard deduction, which is the federal joint figure, instead of NC's $25,500. It also taxed at 4.0% instead of the 3.99% rate for 2026, so it computed $57,201 × 4% = $2,288 rather than $61,701 × 3.99% = $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The rate (3.99%) and joint standard deduction ($25,500) were right. The error was using $79,212 as AGI: it subtracted the spouse's $7,989 employer-sponsored insurance premium from wages as a pre-tax exclusion. The listed gross wages are the wage base, so federal AGI, NC's starting point, is $87,201 and taxable income is $61,701."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"$3,717 is about 4.25% of the full ~$87,000 of gross income, which means it both skipped the $25,500 NC joint standard deduction and used the 2025 rate. The correct calculation taxes $61,701 at the 3.99% rate for 2026."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from $79,212 of AGI, treating the spouse's $7,989 employer-sponsored insurance premium as excluded from wages. Federal AGI is $87,201 (all listed wages plus the $1 of self-employment income), so NC taxable income is $61,701, not $53,712, and 3.99% of it is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used $79,212 as federal AGI by subtracting the spouse's $7,989 employer-sponsored insurance premium from gross wages. No such exclusion applies to the listed wages, so AGI is $87,201. After the $25,500 deduction, $61,701 is taxed at 3.99%, which is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It applied the 2025 rate of 4.25%, and its $3,387.25 implies a taxable base of $79,700. That is far above the correct $61,701 ($87,201 AGI minus the $25,500 NC joint standard deduction), which is taxed at 3.99%."
-us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It began from $79,212 of AGI, treating the spouse's $7,989 employer-sponsored insurance premium as a pre-tax reduction of wages. AGI is actually $87,201, so taxable income after the $25,500 joint standard deduction is $61,701, and 3.99% of it is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave $2,551 with no derivation, about $89 more than the correct $61,701 × 3.99% = $2,461.87. That means it did not apply NC's 3.99% flat rate for 2026 to AGI of $87,201 minus the $25,500 joint standard deduction."
-us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It said this couple has no positive NC liability. But $87,201 of AGI minus the $25,500 joint standard deduction leaves $61,701 of NC taxable income, taxed at the 3.99% flat rate, and no NC nonrefundable credit offsets the resulting $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used $79,212 as NC starting income by excluding the spouse's $7,989 employer-sponsored insurance premium from wages. Federal AGI, NC's starting point, is the full $87,201, so taxable income is $61,701 and the tax is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a 'pre-tax insurance exclusion' that took the spouse's $7,989 employer-sponsored insurance premium out of wages, cutting AGI to about $79,212. The listed gross wages go into AGI in full ($87,201), so NC taxable income is $61,701, and 3.99% of it is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It began from $79,212 of AGI, excluding the spouse's $7,989 employer-sponsored insurance premium from wages. Federal AGI is $87,201, so NC taxable income after the $25,500 joint standard deduction is $61,701, not $53,712, and the 3.99% tax is $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its ~$2,800 matches 4.25% of $65,701, meaning it used the outdated $21,500 joint standard deduction and the 2025 rate. The correct calculation uses NC's $25,500 deduction and the 3.99% rate for 2026: $61,701 × 3.99% = $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected a $26,400 NC joint standard deduction, but the statutory amount is fixed at $25,500. It also used a 4% rate instead of 3.99%, so it computed $60,801 × 4% = $2,432 rather than $61,701 × 3.99% = $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It said no standard deduction applies, but NC allows a $25,500 standard deduction for joint filers. It also cut AGI to $79,212 by excluding the spouse's $7,989 employer-sponsored insurance premium, and taxed at 4% instead of 3.99%. The correct calculation is ($87,201 − $25,500) × 3.99% = $2,461.87."
-us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for state_income_tax_before_refundable_credits, so there was no answer to score against the $2,461.87 correct amount."
-us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a ~$23,400 NC joint standard deduction instead of the statutory $25,500, and the 2025 rate of 4.25% instead of the 3.99% rate for 2026. It taxed $63,801 at 4.25% rather than $61,701 at 3.99%."
-us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2024 rate of 4.5% instead of the 3.99% rate for 2026, and a $23,850 standard deduction instead of NC's $25,500 joint amount. It taxed $63,351 at 4.5% rather than $61,701 at 3.99%."
-us,scenario_102,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It subtracted the $30,000 federal standard deduction instead of NC's own $25,500 joint deduction, and applied a nonexistent 4.49% rate instead of the 3.99% rate for 2026. It computed on $57,201 rather than $61,701 × 3.99% = $2,461.87."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached $61,701 × 3.99% ≈ $2,462. It then dropped that result for a 4.25% rate applied to an unexplained '$64,830 alternative base,' which produced $2,755. Its own reasoning gives no support for either the rate or the base."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed North Carolina has no state income tax. North Carolina has a flat personal income tax, 3.99% in 2026, on federal AGI minus the $25,500 joint standard deduction. That is $61,701 × 3.99% = $2,461.87, not $0."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model applied personal exemptions, which North Carolina does not have. Its $2,700 matches roughly 4.25% on about $63,500 rather than the correct 3.99% on $61,701 (AGI $87,201 minus the $25,500 joint standard deduction)."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The base was right ($61,701), but the model used a 4.25% rate instead of the 3.99% rate for 2026, giving $2,622. It then submitted $2,587, which its own arithmetic does not support."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The reasoning correctly computed $61,701 × 3.99% = $2,462. The model then submitted $3,389, a number that no step of its reasoning produces."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model first reached $61,700 × 3.99% ≈ $2,462. It then replaced that with a guessed 'PolicyEngine parameters' value of $2,792, which equals 4.25% on $65,701, so it brought in both the old 4.25% rate and the outdated $21,500 joint deduction."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $21,500 as the joint standard deduction, but North Carolina's has been $25,500 since 2022. It also used a 4.25% rate instead of the 2026 rate of 3.99%, so it taxed $65,701 at 4.25% rather than $61,701 at 3.99%."
+us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used 4.25% instead of the 2026 rate of 3.99% and got $2,622 on the correct $61,701 base. It then inflated that to $3,846 with unspecified 'adjustments' that its reasoning never supports."
+us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $30,000 joint standard deduction, which is a federal-style figure, instead of North Carolina's $25,500. It also used a 4.0% rate instead of 3.99%, so it taxed $57,201 rather than $61,701."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the spouse's $7,989 employer insurance premium from wages and started from an AGI of $79,212. The stated gross wages already put federal and NC AGI at $87,201, so this understated taxable income by $7,989. The rate (3.99%) and the deduction ($25,500) were right."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used 4.25% instead of the 2026 rate of 3.99%. Its $3,717 equals 4.25% on about $87,459, which means it effectively skipped the $25,500 joint standard deduction that brings taxable income down to $61,701."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $7,989 employer insurance premium as a pre-tax reduction of wages and used an AGI of $79,212 instead of $87,201. That gave $53,712 of taxable income instead of $61,701, though the 3.99% rate and $25,500 deduction were correct."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $79,212, which wrongly removes the $7,989 employer insurance premium from the stated gross wages. AGI is $87,201, and taxable income is $61,701, not $53,712."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used 4.25% instead of 3.99%. Its $3,387.25 equals 4.25% on $79,700, which means it left out the $25,500 joint standard deduction and also took the insurance premium out of income. The correct base is $61,701."
+us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the spouse's $7,989 employer insurance premium from income and started from an AGI of $79,212 instead of $87,201. That understated taxable income by $7,989 even though it used the correct 3.99% rate and $25,500 deduction."
+us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no derivation. Its $2,551 matches roughly 4.25% on about $60,000 rather than the flat 3.99% on $61,701 ($87,201 AGI minus the $25,500 joint standard deduction), so it used the wrong rate for North Carolina."
+us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model concluded the household owes no North Carolina tax. With $87,201 of AGI and a $25,500 standard deduction, $61,701 is taxable at North Carolina's flat 3.99% rate and there are no offsetting nonrefundable credits, so the tax is $2,461.87."
+us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used $79,212 of state income, which removes the spouse's $7,989 employer insurance premium from the stated gross wages. NC taxable income starts from an AGI of $87,201, so taxable income is $61,701, not $53,712."
+us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model explicitly applied a 'pre-tax insurance exclusion' of the $7,989 employer insurance premium, which reduced AGI to $79,212. The stated gross wages are already the AGI wage amount, so taxable income is $61,701 and the tax is $2,461.87."
+us,scenario_102,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $79,212, which wrongly removes the $7,989 employer insurance premium from wages, when AGI is $87,201. With the correct 3.99% rate and $25,500 deduction, taxable income is $61,701, not $53,712."
+us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no workings. Its roughly $2,800 matches 4.25% applied after the outdated $21,500 joint deduction ($65,701 × 4.25%) rather than 3.99% on $61,701."
+us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model projected an inflation-adjusted $26,400 joint standard deduction, but North Carolina's is fixed in statute at $25,500 and is not indexed. It also used a 4% rate instead of 3.99%."
+us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied no North Carolina standard deduction, even though the $25,500 joint deduction applies. It also subtracted the $7,989 employer insurance premium to get $79,212 and used 4% instead of 3.99%."
+us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $2,461.87 result."
+us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of the 2026 rate of 3.99% and an invented $23,400 joint standard deduction instead of North Carolina's $25,500. It therefore taxed $63,801 instead of $61,701."
+us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate (the 2023 rate) instead of the 2026 rate of 3.99%. It also used a $23,850 joint standard deduction instead of North Carolina's $25,500, so it taxed $63,351 at 4.5%."
+us,scenario_102,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model subtracted a $30,000 'federal standard deduction' instead of North Carolina's own $25,500 joint deduction. It also used an invented 4.49% rate instead of 3.99%, which gave $57,201 of taxable income instead of $61,701."
us,scenario_102,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_104,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented automatic eligibility through New York’s medically needy spend-down pathway after its own income calculation exceeded the aged/disabled limit. PolicyEngine assigns no Medicaid category, so neither age nor the existence of a spend-down program converts this household into an eligible Medicaid case."
us,scenario_104,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 72 plus Social Security below an approximate aged/disabled threshold as sufficient and categorically removed all $20,520 of veterans benefits from its income test. The applicable category determination yields NONE, with no SSI-based or other aged pathway establishing eligibility."
@@ -8358,66 +8567,69 @@ us,scenario_108,federal_refundable_credits,grok-build-0.1,llm_error,age_disabili
us,scenario_108,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits. The required result is $0 because every refundable-credit component evaluates to zero, including EITC for this 85-year-old claimant with no qualifying children."
us,scenario_108,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_108,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_108,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model inferred an aged/disabled Medicaid pathway from age, disability, and low assets without establishing a qualifying category. The head matches no Medicaid category, so income exclusions and assets cannot produce eligibility."
-us,scenario_108,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being age 85 and disabled as sufficient categorical qualification for Wisconsin aged/disabled Medicaid. The head matches no Medicaid eligibility category, regardless of its characterization of the income as low."
-us,scenario_108,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly assigned the head to an aged/blind/disabled pathway and then applied SSI-related disregards. Age and disability do not create an engine-recognized category here, and the head receives no SSI."
-us,scenario_108,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented qualification through an aged/disabled or medically needy spend-down pathway, even though the head matches no Medicaid category. The listed $800 of medical expenses, surviving-spouse status, and Indian Health Service coverage do not establish such a pathway."
-us,scenario_108,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed that age, disability, low assets, and asserted income exclusions place the head in Wisconsin’s SSI-related Medicaid pathway. The head receives no SSI and matches no Medicaid category, so that pathway does not apply."
-us,scenario_108,head_medicaid_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model assigned the head to Wisconsin’s Medicaid Purchase Plan solely from disability, work, income, and assets. The head matches no Medicaid category under the applicable PolicyEngine pathways, so the asserted MAPP route cannot support eligibility."
-us,scenario_108,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used elderly status, low income, and minimal assets as a shortcut to eligibility without identifying a qualifying Medicaid pathway. The head matches no Medicaid category, which ends the eligibility determination."
-us,scenario_108,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated age, disability, and Indian Health Service coverage as establishing Medicaid eligibility. None supplies a qualifying Medicaid category for this head, and IHS coverage is not an eligibility pathway."
-us,scenario_108,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model applied earned-income exclusions under an elderly/disabled budgeting pathway without first establishing categorical eligibility. The head matches no Medicaid category, so favorable income budgeting cannot make the head eligible."
-us,scenario_108,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility directly from disability, senior status, low income, and IHS coverage. Those facts do not place the head in any Medicaid category, and IHS coverage does not create one."
-us,scenario_108,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly placed the head in an SSI-related aged/blind/disabled category based on age, disability, IHS coverage, and its income estimate. The head receives no SSI and matches no Medicaid category; it also omitted the $7,656 of wages when describing countable income."
+us,scenario_108,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It added up $30,982 of income and named a 100% FPL limit of about $13,590, then still declared the head eligible. Its own numbers show income more than double the limit. The Social Security exclusion it relied on applies to tax MAGI, not to aged/disabled countable income, so the head fails the aged/blind/disabled income test."
+us,scenario_108,head_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It called the head's income 'low' without comparing it to any limit. Gross income of $30,982 (wages plus $23,326 in survivor benefits) is about twice the FPL and far above Wisconsin's aged/blind/disabled income standard. With SSI at $0, being 85 and disabled does not open any Medicaid category."
+us,scenario_108,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It said income was 'well below limits' without adding up the $12,934 SS survivor benefit and the $10,392 survivor benefit. Countable unearned income alone is about $1,944 a month, well above Wisconsin's aged/disabled income standard, so the head fails the only non-MAGI pathway open to someone aged 85."
+us,scenario_108,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It correctly picked SSI-related aged/blind/disabled budgeting but called the income 'very low' without doing the math. After the $20 general and $65-plus-half earned-income disregards, countable income is about $2,200 a month. That exceeds both the SSI benefit rate (SSI = $0) and the aged/blind/disabled income limit, so no category applies."
+us,scenario_108,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It left out the $10,392 survivor benefit, then admitted income could exceed the aged/blind/disabled standard and relied on a medically needy spend-down pathway that PolicyEngine's Medicaid categories do not include. Even under spend-down, $800 of medical expenses would not bring $30,982 of income down to the limit. IHS coverage and surviving-spouse status are not eligibility categories."
+us,scenario_108,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted only wages and the $12,934 SS survivor benefit, called the result 'very low,' and left out the $10,392 survivor benefit. Total countable income is about twice the FPL, which exceeds Wisconsin's elderly/blind/disabled income limit, and SSI is $0, so the head qualifies under no category."
+us,scenario_108,head_medicaid_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It based eligibility on Wisconsin's Medicaid Purchase Plan (the 250% FPL buy-in for working people with disabilities), which is not one of the Medicaid categories PolicyEngine evaluates. Under the categories PolicyEngine does use, the head is too old for the adult group and over the aged/blind/disabled income limit."
+us,scenario_108,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It called the income 'very low' without checking it against a limit. The head has $30,982 of gross income, about double the FPL, which is far over Wisconsin's aged/blind/disabled income standard. Low assets ($270) satisfy the asset test, but they cannot make up for failing the income test."
+us,scenario_108,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It treated age, disability, and Indian Health Service coverage as qualifying in themselves. IHS coverage is not a Medicaid eligibility category, and the aged/disabled pathway requires countable income under the state's standard. The head's roughly $31,000 of income fails that test."
+us,scenario_108,head_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It applied the earned-income exclusions, but those only reduce the $7,656 of wages. The $23,326 of unearned survivor benefits gets only the $20 disregard, so countable income stays near $2,200 a month, above Wisconsin's elderly/disabled limit. The head is ineligible."
+us,scenario_108,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It treated being a disabled senior with IHS coverage as enough for Medicaid. IHS coverage gives no Medicaid eligibility, and the aged/disabled pathway requires income below the state standard. The head's $30,982 total income is about twice the FPL and fails that test."
+us,scenario_108,head_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It counted $23,326 of survivor benefits and said that was within SSI-related Medicaid limits, but $23,326 a year (about $1,944 a month) is far above both the SSI benefit rate (SSI = $0) and Wisconsin's aged/blind/disabled income limit. It also left out the $7,656 in wages."
us,scenario_108,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_108,payroll_tax,claude-sonnet-4.6,llm_error,other,False,"The model transcribed the Medicare calculation as $110.01 even though its own displayed multiplication, 1.45% × $7,656, equals $111.01. Adding $474.67 and $111.01 yields $585.68, exactly $1 above its answer."
-us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified approximately $475 of Social Security tax and $111 of Medicare tax, which sum to about $586, but then submitted $605 with no additional payroll-tax component. Using the exact components gives $474.67 + $111.01 = $585.68."
-us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract."
+us,scenario_108,payroll_tax,claude-haiku-5.5,llm_error,other,False,"The model correctly worked out 7.65% × $7,656 = $585.68 in its reasoning, then wrote 'value = 586' in its explanation and submitted 575. It didn't carry its own correct FICA total into the value field and entered a number that matches nothing it computed."
+us,scenario_108,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"It got the Medicare tax wrong: 0.0145 × $7,656 is $111.012, but the model wrote $110.012 (and $110.01), so its answer was $1 too low. Its Social Security figure ($474.67) and its exclusions (no Additional Medicare Tax, no Wisconsin payroll tax) were correct."
+us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly worked out about $475 in Social Security tax plus $111 in Medicare tax, roughly $586, then gave $605 as its final answer for no stated reason. The $19 increase has no basis in any payroll tax rule, because Wisconsin has no employee state payroll tax and Additional Medicare Tax does not apply."
+us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value and no reasoning, so there was no answer to score. The correct derivation is 7.65% employee FICA on $7,656 of wages, which gives $585.68."
us,scenario_108,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_108,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income test (~$1,305) to a net income of ~$2,217 and declared the household ineligible. It missed that Wisconsin's TANF non-cash broad-based categorical eligibility waives the net income test. An eligible one-person household whose formula benefit is negative gets the $24 monthly minimum, or $288 a year."
-us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income of ~$2,214 exceeded the 100% FPL net limit. It ignored Wisconsin's broad-based categorical eligibility, which removes the net income test. The categorically eligible one-person household gets the $24 monthly minimum allotment ($288 a year)."
-us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It tested countable income against a 130% FPL limit and declared the household ineligible. But elderly/disabled households are not subject to the 130% gross test. Wisconsin's BBCE gross limit is 200% FPL (~$2,608), which the $2,582 gross income passes. The categorical eligibility that follows yields the $24 monthly minimum benefit, $288 a year."
-us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"Its own arithmetic showed max allotment minus 30% of net income was negative, and it reached a minimum-benefit answer of $23/month ($276/yr). It then discarded that for an unsupported $2,227 that none of its computations produce. The correct result is the FY2026 minimum of $24/month, $288/yr."
-us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning applied the 100% FPL net income test (~$1,304) to ~$2,186 net income and concluded the household is ineligible. It never recognized that Wisconsin's BBCE waives that test. It then submitted $2,376, which contradicts its own ineligibility conclusion. The categorically eligible household gets the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly landed on a minimum-level benefit but used the FY2025 minimum allotment of $23/month ($276/yr). The FY2026 minimum for a one-person household is $24/month ($288/yr). It also overstated the excess medical deduction as ~$465/month. The monthly medical cost is $66.67, so the deduction is only $31.67."
-us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It asserted that Wisconsin has no broad-based categorical eligibility and applied the 100% FPL net income test to $2,224 net income, concluding the household is ineligible. Wisconsin does confer categorical eligibility through TANF non-cash benefits, which removes the net income test. The household therefore gets the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It claimed an excess shelter deduction that brought net income down to ~$1,700 and yielded $164/month. But $600 rent is below half of adjusted income, so the shelter deduction is $0. Net income stays near $2,036, and 30% of it ($611) exceeds the $298 maximum, so only the $24 minimum ($288/yr) is paid."
-us,scenario_108,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the 100% FPL net income test as binding for the elderly/disabled household and returned $0. Wisconsin's BBCE via TANF non-cash waives that test. The eligible one-person household whose formula benefit is negative receives the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It declared the household ineligible because ~$2,184 net income exceeds the 100% FPL net income limit. It missed that Wisconsin's broad-based categorical eligibility removes the net income test. The categorically eligible one-person household receives the $24/month minimum, $288/yr."
-us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It stopped at a negative formula benefit and returned $0. It missed that the household is categorically eligible through Wisconsin's TANF non-cash BBCE, and that eligible one- and two-person households get a minimum allotment. The minimum is $24/month, or $288/yr."
-us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It reasoned that 30% of net income exceeds the one-person maximum allotment and set the benefit to $0. It omitted the minimum allotment that SNAP pays to eligible one- and two-person households. This household is categorically eligible through Wisconsin BBCE, so it receives $24/month ($288/yr)."
-us,scenario_108,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit for elderly/disabled households and returned $0. Wisconsin's broad-based categorical eligibility waives the net income test for this household. The benefit is therefore the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It said gross income exceeds the limits and that 30% of net income exceeds the maximum allotment, then set the benefit to $0. It missed two things: gross income of $2,581.83 is under Wisconsin's 200% FPL BBCE limit, and categorically eligible one-person households get the $24/month minimum ($288/yr)."
-us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared income above the SNAP eligibility threshold without further analysis. The $2,581.83 gross income passes Wisconsin's 200% FPL BBCE limit for this elderly/disabled household. Categorical eligibility waives the net income test, so the household gets the $24/month minimum allotment, $288/yr."
-us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It asserted that both gross and net income exceed SNAP limits. It never applied Wisconsin's broad-based categorical eligibility: the 200% FPL gross limit, which the $2,581.83 gross income passes, and no net income test. Under that pathway the one-person household receives the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly found categorical eligibility through Wisconsin's 200% FPL BBCE and correctly applied the minimum benefit. But it used the FY2025 minimum of $23/month ($276/yr). The FY2026 minimum allotment for a one-person household is $24/month, or $288/yr."
-us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It asserted a zero benefit from income and resources. The $270 in assets pass the asset test, and the $2,581.83 gross income passes Wisconsin's 200% FPL BBCE limit, so the household is categorically eligible. Because the formula benefit is negative, it receives the $24/month minimum, $288/yr."
-us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It treated $2,581.83 of monthly gross income as over the SNAP limits and set the benefit to $0. That income is within Wisconsin's 200% FPL BBCE gross limit, and categorical eligibility removes the net income test. When the formula benefit goes negative, the minimum allotment of $24/month ($288/yr) applies."
-us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It applied a net income threshold to rule the household out. Wisconsin's TANF non-cash categorical eligibility waives the net income test for this household. The eligible one-person household receives the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the one-person maximum allotment, but then set the benefit to $0. It omitted the minimum allotment for eligible one- and two-person households. This household qualifies through Wisconsin BBCE, so the benefit is $24/month, $288/yr."
-us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,"It computed max allotment minus 30% of ~$2,196 net income as negative and returned $0. It never applied the minimum allotment for categorically eligible one-person households. With Wisconsin BBCE eligibility, that minimum is $24/month, $288/yr."
-us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"It ruled the household out because net income is above the 100% FPL net limit. It also wrongly described the shelter deduction as capped, although it is uncapped for elderly/disabled households. Wisconsin's BBCE waives the net income test, so the household gets the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted a substantial benefit of $1,172/yr without computing net income. After the earned income, standard, and medical deductions, and with no excess shelter deduction, net income is about $2,036. Thirty percent of that ($611) exceeds the $298 maximum, so the benefit is only the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It set SNAP to $0 on the basis of the household's income without applying Wisconsin's broad-based categorical eligibility. That pathway passes the household at 200% FPL gross and waives the net income test. The categorically eligible one-person household receives the $24/month minimum allotment, $288/yr."
-us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It concluded that income is too high to produce a positive allotment and returned $0. It omitted the minimum allotment that SNAP guarantees to eligible one- and two-person households. The household is categorically eligible through Wisconsin BBCE, so it receives $24/month, $288/yr."
-us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It produced $103/month by overstating the deductions, apparently crediting the $600 rent as a shelter deduction. But rent is below half of adjusted income, so the excess shelter deduction is $0. Net income is about $2,036, 30% of it ($611) exceeds the $298 maximum, and only the $24 minimum ($288/yr) is paid."
-us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It declared the household ineligible because net income exceeds the one-person elderly/disabled net income limit. Wisconsin's TANF non-cash categorical eligibility waives the net income test. The household therefore gets the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It applied the elderly/disabled net income limit to rule the household ineligible. Wisconsin's broad-based categorical eligibility removes the net income test. The eligible one-person household's negative formula benefit is raised to the $24 monthly minimum, $288/yr."
-us,scenario_108,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It ruled the household out on the SNAP net income limit. Wisconsin's BBCE via TANF non-cash benefits makes this household categorically eligible without a net income test. Its benefit is the $24/month minimum allotment, $288/yr."
-us,scenario_108,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It concluded that net income is too high for a one-person allotment and returned $0. It skipped the minimum allotment that SNAP pays to eligible one- and two-person households. The household is categorically eligible through Wisconsin BBCE, so it gets $24/month, $288/yr."
-us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"It stated that income exceeds SNAP limits after deductions and returned $0. Wisconsin's broad-based categorical eligibility waives the net income test, and gross income is within its 200% FPL limit. The household receives the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"It compared ~$28,000 of annual net income with the 100% FPL net income limit and returned $0. That test does not apply to a household categorically eligible through Wisconsin's TANF non-cash BBCE. The benefit is the $24/month minimum, $288/yr."
-us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"It waived the gross test for the elderly/disabled household but failed it on the 100% FPL net income test. Wisconsin's BBCE removes that net income test. The eligible one-person household therefore receives the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,grok-4.7,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income exceeds the one-person net income limit. It missed Wisconsin's broad-based categorical eligibility, which removes that test. The minimum allotment of $24/month ($288/yr) then applies."
-us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It computed ~$2,218 net income against the ~$1,305 100% FPL net limit and returned $0. It did not recognize that Wisconsin's TANF non-cash categorical eligibility waives the net income test. The eligible household receives the $24/month minimum, $288/yr."
-us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"It failed the household because ~$2,225 net income exceeds the ~$1,300 100% FPL limit. Wisconsin's broad-based categorical eligibility waives the net income test. The eligible household gets the $24 monthly minimum allotment, $288/yr."
-us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit for elderly/disabled households and declared the household ineligible. Wisconsin's TANF non-cash BBCE makes the household categorically eligible without that test. The benefit is the $24/month minimum allotment, $288/yr."
-us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly found that 30% of ~$2,214 net income far exceeds the maximum allotment, but then set the benefit to $0. It omitted the minimum allotment for eligible one- and two-person households. With categorical eligibility through Wisconsin BBCE, the household receives $24/month, $288/yr."
-us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross test, which elderly/disabled households are exempt from, and the 100% FPL net test to rule the household out. It also left out the earned income deduction. Wisconsin's BBCE sets a 200% FPL gross limit, which the $2,581.83 gross income passes, and has no net test. The eligible household gets the $24/month minimum, $288/yr."
-us,scenario_108,snap,ox-alpha,llm_error,categorical_eligibility,False,"It applied a 165% FPL gross income test, the separate-purchase-and-preparation rule for elderly/disabled members, and ruled the household ineligible. The household is categorically eligible through Wisconsin's BBCE, whose 200% FPL gross limit (~$2,608) the $2,582 gross income passes. It gets the $24/month minimum, $288/yr."
-us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Its reasoning noted that a contribution above the maximum allotment means SNAP 'would be minimum,' but its final answer treated the household as not eligible and submitted $0. Along the way it also invented a ~$400 utility allowance that the facts do not support. Categorical eligibility through Wisconsin BBCE yields the $24/month minimum allotment, $288/yr."
-us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It declared countable income above the one-person SNAP limit and returned $0. Wisconsin's broad-based categorical eligibility applies a 200% FPL gross limit, which this household passes, and no net income test. The eligible household receives the $24 monthly minimum allotment, $288/yr."
+us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to its ~$2,217 net income and declared the household ineligible. It missed that Wisconsin BBCE (TANF non-cash categorical eligibility, 200% FPL gross limit) waives the net income test. A categorically eligible one-person household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Ruled the household ineligible because ~$2,214 net income exceeds the ~$1,305 net limit. It ignored Wisconsin BBCE: gross income of $2,581.83 is under 200% FPL, so the household is categorically eligible and the net income test is waived. Its zero formula benefit therefore becomes the $24/month one-person minimum allotment."
+us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Tested countable income against a 130% FPL limit, which does not govern here. Wisconsin BBCE uses a 200% FPL gross limit that $2,581.83/month passes, and elderly/disabled households are exempt from the 130% test anyway. The household is categorically eligible and gets the $24/month minimum allotment."
+us,scenario_108,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Asserted a zero benefit without any computation and never applied the minimum-allotment rule. The household is categorically eligible through Wisconsin BBCE, so even though its formula benefit is zero, it receives the $24/month one-person minimum ($288/yr)."
+us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"Its own reasoning reached the minimum-benefit outcome, then it discarded that and submitted an unsupported $2,227. The correct result is the minimum allotment for a categorically eligible one-person household: 30% of net income ($611) exceeds the $298 maximum. The FY2026 minimum is $24/month ($288/yr), not the $23 it cited."
+us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning concluded the household fails the 100% FPL net income test, but it submitted an unrelated $2,376. Both steps are wrong: Wisconsin BBCE waives the net income test, and a categorically eligible one-person household whose formula benefit is negative gets only the $24/month minimum ($288/yr)."
+us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"It reached the minimum-allotment outcome but used the outdated $23 minimum. The FY2026 one-person minimum is 8% of the $298 maximum, rounded to $24/month, which gives $288/yr instead of its $276. Its reasoning also overstated the medical deduction (~$465 instead of ~$32/month)."
+us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Explicitly asserted that Wisconsin has no broad-based categorical eligibility and then failed the household on the net income test. Wisconsin FoodShare does use BBCE at 200% FPL gross, which $2,581.83/month passes. That waives the net income test, and the household receives the $24/month minimum allotment."
+us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Assumed net income of about $1,700 and a $164/month regular benefit, an unsupported excess shelter deduction. Actual net income is about $2,036, so the 30% contribution of $611 exceeds the $298 maximum and the formula benefit is zero. Only the $24/month categorical-eligibility minimum applies ($288/yr)."
+us,scenario_108,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test and declared the household ineligible. Wisconsin BBCE (gross income under 200% FPL) makes the household categorically eligible, which waives the net income test and guarantees the $24/month one-person minimum allotment."
+us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Failed the household on the 100% FPL net income test using net income of ~$2,184. It missed Wisconsin's TANF non-cash BBCE pathway, which waives that test, and the rule that categorically eligible one-person households get the $24/month minimum allotment."
+us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment, but treated the negative result as $0. Under Wisconsin BBCE the household is categorically eligible, so a one-person household receives the $24/month minimum allotment ($288/yr) instead of zero."
+us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Found that the 30% contribution exceeds the one-person maximum and stopped at zero. It never applied the minimum-benefit rule: categorically eligible one- and two-person households receive $24/month in FY2026, which gives $288/yr."
+us,scenario_108,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to an elderly/disabled household. Wisconsin BBCE confers categorical eligibility that waives this test, and the household receives the $24/month one-person minimum allotment."
+us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Claimed gross income exceeds SNAP limits and zeroed the benefit because 30% of net income exceeds the maximum. Gross income of $2,581.83 is under Wisconsin's 200% FPL BBCE limit, so the household is categorically eligible and receives the $24/month minimum allotment."
+us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Asserted that income exceeds the eligibility threshold, but $2,581.83/month passes Wisconsin's 200% FPL BBCE gross limit. Through that categorical eligibility, the one-person household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Claimed both the gross and net income limits are exceeded. Wisconsin BBCE sets the gross limit at 200% FPL (about $2,608), which this household passes, and categorical eligibility waives the net income test. The household therefore gets the $24/month minimum allotment."
+us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly identified Wisconsin BBCE eligibility and the minimum-benefit outcome, but used the outdated $23 minimum. The FY2026 one-person minimum is $24/month (8% of the $298 maximum), so the annual amount is $288, not $276."
+us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Returned zero without applying the minimum-allotment rule. The household is categorically eligible through Wisconsin BBCE, so although the formula benefit is negative, the one-person household receives $24/month ($288/yr)."
+us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Claimed $2,581.83/month gross income exceeds SNAP limits, but it passes Wisconsin's 200% FPL BBCE gross test. It also reduced the benefit to $0 instead of applying the $24/month minimum allotment for categorically eligible one-person households."
+us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Applied a net income threshold and zeroed the benefit. Wisconsin BBCE categorical eligibility waives the net income test, and the one-person household receives the $24/month minimum allotment."
+us,scenario_108,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum, but stopped at zero. A categorically eligible one-person household receives the FY2026 minimum allotment of $24/month instead of zero."
+us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,Computed max allotment minus 30% of net income as negative and returned $0. It missed the minimum-benefit rule: categorically eligible one-person households under Wisconsin BBCE receive $24/month ($288/yr).
+us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test, and applied a shelter cap that does not apply to elderly/disabled households. Wisconsin BBCE waives the net income test, so the household is eligible and gets the $24/month minimum allotment."
+us,scenario_108,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Submitted an unsupported $1,172 (about $98/month) regular benefit. Net income of about $2,036 makes the 30% contribution ($611) exceed the $298 maximum, so the formula benefit is zero. The household gets only the $24/month categorical-eligibility minimum ($288/yr)."
+us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,Defaulted to zero without applying Wisconsin's BBCE pathway or the minimum-allotment rule. A categorically eligible one-person household with a zero formula benefit receives $24/month ($288/yr).
+us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Concluded that income is too high to produce a positive allotment and returned zero. Under Wisconsin BBCE the household is categorically eligible, so the one-person minimum allotment of $24/month applies."
+us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Estimated an unsupported $103/month regular benefit. Rent of $600 adds little or no shelter deduction against adjusted income of about $2,214, and net income of about $2,036 makes the 30% contribution ($611) exceed the $298 maximum. The household gets only the $24/month minimum ($288/yr)."
+us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Applied the elderly/disabled 100% FPL net income limit and declared the household ineligible. Wisconsin BBCE waives the net income test, and the categorically eligible one-person household receives the $24/month minimum allotment."
+us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Failed the household on the net income limit, missing that Wisconsin's TANF non-cash BBCE confers categorical eligibility that waives it. The one-person household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_108,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Applied the SNAP net income limit and returned zero. Under Wisconsin BBCE the household is categorically eligible, which waives the net income test and entitles the one-person household to the $24/month minimum."
+us,scenario_108,snap,gpt-6-sol,llm_error,categorical_eligibility,False,Concluded that net income is too high for any allotment and returned zero. Categorically eligible one-person households get the minimum allotment of $24/month in FY2026 even when the formula benefit is negative.
+us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"Asserted that income exceeds the SNAP limits after deductions. Wisconsin BBCE (200% FPL gross) makes the household categorically eligible, and the one-person minimum allotment of $24/month applies."
+us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"Compared annual net income to 100% FPL and declared the household ineligible. Wisconsin BBCE waives the net income test for this household, which then receives the $24/month minimum allotment ($288/yr)."
+us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"Treated the 100% FPL net income test as binding. Wisconsin BBCE (gross income under 200% FPL) confers categorical eligibility that removes it, leaving the household with the $24/month one-person minimum allotment."
+us,scenario_108,snap,grok-4.7,llm_error,categorical_eligibility,False,"Failed the household on the net income limit. It missed the Wisconsin BBCE pathway, which waives that limit, and the $24/month minimum allotment for categorically eligible one-person households."
+us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Computed net income of about $2,218 and failed the household against the ~$1,305 net limit. Wisconsin BBCE categorical eligibility waives the net income test, and the one-person household receives the $24/month minimum allotment."
+us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to ~$2,225 net income and returned zero. The household is categorically eligible under Wisconsin BBCE, so the net test is waived and the $24/month one-person minimum applies."
+us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Declared the household ineligible under the 100% FPL net income limit. Wisconsin's TANF non-cash BBCE waives that limit, and the one-person household receives the $24/month minimum allotment."
+us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income far exceeds the one-person maximum, but returned $0. Categorically eligible one-person households receive the FY2026 minimum allotment of $24/month ($288/yr)."
+us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross test and a 100% FPL net test, and omitted the earned income deduction. Wisconsin BBCE uses a 200% FPL gross limit that the household passes and waives the net test, so the household gets the $24/month minimum allotment."
+us,scenario_108,snap,ox-alpha,llm_error,thresholds_rates,False,"Applied a 165% FPL gross income limit. Wisconsin BBCE sets the limit at 200% FPL (about $2,608/month), which $2,581.83 passes. The household is categorically eligible and receives the $24/month minimum allotment."
+us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Noted that 'SNAP would be minimum' when the 30% contribution exceeded the maximum, then returned $0 anyway. Categorically eligible one-person households receive the $24/month minimum allotment ($288/yr)."
+us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Asserted that countable income exceeds the SNAP limit. Wisconsin BBCE (200% FPL gross) makes the household categorically eligible regardless of the net income test, and it receives the $24/month one-person minimum allotment."
us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model stopped at a gross tax of $134.33 (3.5% of $3,838) and never subtracted Wisconsin's nonrefundable renter school property tax credit. On the $7,200 rent that credit is 12% of $1,440-$1,800, or $172.80-$216, which wipes out the tax. It also used a made-up $270 age deduction instead of Wisconsin's $700 personal exemption plus the $250 exemption for age 65 and over."
us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model said outright that there were 'no other nonrefundable credits', which skips the renter school property tax credit. That credit is 12% of 20-25% of the $7,200 rent, or $172.80-$216, and it takes the $161.71 gross tax to zero. It also used an old 3.54% bottom rate instead of the current 3.50% and left out the $700 and $250 exemptions."
us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the bottom rate to about $4,548 of taxable income and reported the $159 gross tax as the final answer. It never subtracted the nonrefundable renter school property tax credit ($172.80-$216 on $7,200 rent), which brings Wisconsin tax before refundable credits to $0."
@@ -8427,52 +8639,53 @@ us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_erro
us,scenario_108,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model taxed $3,538 at 3.5% to get $124 and stopped. It never subtracted Wisconsin's nonrefundable renter school property tax credit, which is $172.80-$216 on $7,200 of rent and is larger than $124, so the tax before refundable credits is $0."
us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said outright that there were 'no nonrefundable state credits', so it skipped the renter school property tax credit ($172.80-$216 on $7,200 rent) that takes its $150 tax to zero. It also used an old 3.54% bottom rate instead of 3.50% and left out the $700 and $250 exemptions."
us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to grade. The correct derivation takes a small gross Wisconsin tax and applies the nonrefundable renter school property tax credit, which brings it to $0."
-us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in homestead income and concluded the credit was zero. It then submitted an unsupported $636 'partial credit', using 25% of rent capped at $1,460 instead of the 20% rent factor ($1,440). With homestead income of wages plus Social Security ($20,590), the formula 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) gives $271.39."
-us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits in Wisconsin homestead household income, reaching $30,982, and zeroed the credit at the $24,680 limit. Homestead income here is wages plus nontaxable Social Security, or $20,590. That is under the cap and yields a $271.39 credit on $1,440 of rent-based property taxes."
-us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It computed the correct $20,590 household income but dismissed the Wisconsin homestead credit as 'property tax relief, not a refundable income tax credit'. The homestead credit is a refundable credit claimed with the Wisconsin return (Schedule H) and counts toward state refundable credits. Applying it gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It used the correct $1,440 rent-based property tax but built household income as wages plus survivor benefits ($18,048). It dropped the $12,934 of Social Security, which homestead income includes, and added survivor benefits, which it does not. It also eyeballed $300 instead of applying 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security ($30,982 total) and ruled the homestead credit out at the $24,680 limit. Homestead household income is $7,656 + $12,934 = $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It used a household income of about $18,000, which matches wages plus survivor benefits. That base excludes the $12,934 of Social Security that homestead income includes and wrongly counts the survivor benefits. Its $371 does not follow from any consistent formula. The correct $20,590 base gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It set homestead household income at $30,982 by including the $10,392 survivor benefits, so the household appeared to exceed the $24,680 limit. Homestead income is wages plus Social Security ($20,590), which yields a $271.39 credit."
-us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It explicitly added 'other survivor benefits ($10,392)' to wages and Social Security, reaching $30,982, and zeroed the homestead credit. Homestead household income excludes that input and equals $20,590. That is below the cap and produces 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It named the homestead credit but never computed it, claiming no refundable-credit trigger without children. An 85-year-old disabled renter with $7,200 of rent and $20,590 of homestead income qualifies. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used the correct $20,590 income but set rent-based property taxes at 25% of rent capped at $1,460, instead of 20% of rent ($1,440). It also used an 8.65% reduction rate above $8,000 instead of 8.785% above $8,060. Its 0.8 × (1,460 − 0.0865 × 12,590) = $297 should be 0.8 × ($1,440 − 0.08785 × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It declared household income above the homestead limit, which requires adding the $10,392 survivor benefits. Homestead household income is wages plus Social Security ($20,590), under the $24,680 cap. The credit is $271.39."
-us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income by counting the $10,392 survivor benefits and zeroed the credit. The correct homestead income is $7,656 + $12,934 = $20,590. That gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It concluded household income exceeds the homestead limit by folding in the $10,392 survivor benefits. Homestead income counts only wages and Social Security here ($20,590), which is below $24,680. That produces a $271.39 credit."
-us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It returned $0 with no analysis and never applied the Wisconsin homestead credit. The claimant is 85, disabled, pays $7,200 in rent and has $20,590 of homestead income. Those facts yield 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as part of AGI, getting $30,982 of 'AGI plus Social Security', and applied the $24,680 cutoff. AGI is only the $7,656 of wages, so homestead income is $20,590. The credit is $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It asserted no Wisconsin refundable-credit eligibility without testing the homestead credit. An elderly, disabled renter with $20,590 of homestead income and $1,440 of rent-based property taxes receives $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It computed $30,982 of household income by including the $10,392 survivor benefits and ruled out the homestead credit. Homestead income is wages plus Social Security ($20,590), under the limit. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, reaching $30,982 against the $24,680 limit. Homestead household income is $20,590 (wages plus Social Security), so the credit is $271.39 rather than $0."
-us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It declared the head ineligible for the homestead credit without any income test. The head is 85 and disabled, pays rent, and has $20,590 of homestead income, below $24,680. The credit is $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It used $30,982 of total household income, including the $10,392 survivor benefits, and applied the $24,680 threshold. Homestead income is only the $7,656 of wages plus $12,934 of Social Security, or $20,590. That yields $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Its 'income exceeds the homestead limit' conclusion depends on adding the $10,392 survivor benefits. Homestead household income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It placed household income above the $24,680 homestead limit by counting the survivor benefits. Homestead income is wages plus Social Security ($20,590), which yields a $271.39 credit on $1,440 of rent-based property taxes."
-us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used $30,982 of household income by adding the $10,392 survivor benefits and zeroed the homestead credit. The correct homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits as pension income, reaching $30,982 against a ~$24,780 limit. Homestead household income here is wages plus Social Security only ($20,590). The credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It found no Wisconsin refundable credit and never considered the homestead credit. The 85-year-old disabled renter with $7,200 of rent and $20,590 of homestead income receives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It looked only for EITC-like credits and omitted the refundable Wisconsin homestead credit. With rent-based property taxes of $1,440 and homestead income of $20,590, that credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It included the listed survivor benefits with Social Security and wages, which pushed household income over the homestead limit. Homestead income excludes the $10,392 survivor-benefits input and equals $20,590. The credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It concluded no refundable credit applies to an elderly taxpayer without children and overlooked the homestead credit. That credit does not require children and pays $271.39 on $1,440 of rent-based property taxes and $20,590 of household income."
-us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It placed income above the homestead limit, which reflects counting the $10,392 survivor benefits. Homestead household income is $7,656 + $12,934 = $20,590. That gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It found no refundable-credit eligibility for a one-adult household and never evaluated the homestead credit. The head is 85, disabled and renting, with $20,590 of homestead income, which yields $271.39."
-us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It used $30,982 of total income, including the $10,392 survivor benefits, against a $24,480 ceiling. Homestead household income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It ruled out the homestead credit on income by counting the survivor benefits. Homestead income is wages plus Social Security ($20,590), below $24,680. The credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It treated 'total household income', including the $10,392 survivor benefits, as the homestead income base. The base is $20,590, and the formula 0.8 × ($1,440 − 8.785% × $12,530) gives $271.39."
-us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It concluded total household income exceeds the homestead limit by including the $10,392 survivor benefits. Homestead income is $7,656 + $12,934 = $20,590, which is under the cap. The credit is $271.39."
-us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It found no qualifying state refundable credit and skipped the Wisconsin homestead credit. The household qualifies with $1,440 of rent-based property taxes and $20,590 of homestead income, for a $271.39 credit."
-us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used ~$30,982 of household income by adding the $10,392 survivor benefits and compared it to a ~$25,000 limit. Homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It counted Social Security and the survivor benefits for $30,982 of homestead income and zeroed the credit. The $10,392 survivor-benefits input is not homestead income, so income is $20,590. The credit is $271.39."
-us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly summed wages, Social Security survivor benefits and other survivor benefits to exceed the homestead limit. Homestead household income excludes the $10,392 survivor benefits, giving $20,590 and a credit of $271.39."
-us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It treated the survivor benefits as AGI, computing 'AGI + SS = $30,982' against a ~$26,500 limit. AGI is only the $7,656 of wages, so homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It evaluated only the Wisconsin EITC and never applied the refundable homestead credit. For this 85-year-old disabled renter with $20,590 of homestead income, that credit is $271.39."
-us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_refundable_credits, so there is no answer to score against the $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a 'broad' household income of $30,982 that includes the $10,392 survivor benefits and ruled out the homestead credit. Homestead income is wages plus Social Security ($20,590), yielding $271.39."
-us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,"It reasoned that zero Wisconsin tax liability means no refundable credits. Refundable credits are paid regardless of liability, and the homestead credit here is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
-us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits along with wages and Social Security ($30,982) and ruled out the homestead credit. Homestead income is $20,590, below the $24,680 cap. The credit is $271.39."
-us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It explicitly added the $10,392 survivor benefits to wages and Social Security survivor benefits for $30,982 and applied the $24,680 limit. Homestead household income is $20,590. That yields 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
-us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It reasoned only from the lack of qualifying children and omitted the homestead credit, which needs no children. An 85-year-old disabled renter with $20,590 of homestead income receives $271.39."
+us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in household income instead of $20,590 (wages plus Social Security). It also took 25% of rent capped at $1,460 instead of 20% of rent ($1,440). Its $636 does not follow from any formula it stated; the correct calculation, 0.8 × ($1,440 − 8.785% × $12,530), gives $271.39."
+us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It added the separate $10,392 survivor benefits, putting household income at $30,982 and over the $24,680 limit. Homestead household income is wages plus Social Security, $20,590, which qualifies for a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It correctly found $20,590 in income but wrongly said the Homestead Credit is not a refundable income tax credit. The credit is claimed on the Wisconsin return and is refundable: 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
+us,scenario_108,state_refundable_credits,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It reasoned that refundable credits require Wisconsin tax liability to offset. The Homestead Credit is paid in full even with no liability, and this 85-year-old disabled renter with $20,590 in household income gets $271.39."
+us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It wrongly left Social Security out of household income and wrongly added the $10,392 survivor benefits, getting $18,048 instead of $20,590. It then guessed $300 rather than applying 0.8 × ($1,440 − 8.785% × (income − $8,060)), even though it used the correct $1,440 rent figure."
+us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It counted the separate $10,392 survivor benefits in household income ($30,982 total) and concluded the household was over the $24,680 limit. Wages plus Social Security total $20,590, which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Its income of about $18,000 means it dropped the $12,934 in Social Security and counted the $10,392 survivor benefits. The correct $20,590 gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39, not $371."
+us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, making household income $30,982 instead of $20,590, and wrongly ruled out the credit under the $24,680 limit."
+us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It added the $10,392 in other survivor benefits to wages and Social Security, getting $30,982. Homestead household income here is $20,590, which is under the limit and gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It dismissed the Homestead Credit with a vague appeal to income level and age and never calculated it. An 85-year-old disabled renter with $20,590 in household income and $1,440 in rent-based property taxes qualifies for $271.39."
+us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used the correct $20,590 income but took 25% of rent capped at $1,460 instead of 20% of rent ($1,440 when heat is not included). It also used an 8.65% rate and an $8,000 threshold instead of 8.785% and $8,060, which gave $297 instead of $271.39."
+us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit, which only happens if the $10,392 survivor benefits are counted. Wages plus Social Security total $20,590, which is under the $24,680 limit and gives $271.39."
+us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Its $30,982 income figure includes the separate $10,392 survivor benefits. Homestead household income is $20,590, which qualifies for a $271.39 credit."
+us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit because it counted the $10,392 survivor benefits on top of wages and Social Security. The correct $20,590 gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It never considered the Homestead Credit, which this 85-year-old disabled renter with $20,590 in household income and $7,200 in rent qualifies for. It is worth 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
+us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It put household income at $30,982 by including the $10,392 survivor benefits. Wages plus Social Security total $20,590, which is under the $24,680 limit and gives $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It said the household was ineligible without calculating anything and skipped the Homestead Credit. With $20,590 in household income and $1,440 in rent-based property taxes, the credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Its $30,982 total includes the separate $10,392 survivor benefits. Homestead household income is $20,590 (wages plus Social Security), which qualifies for $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, got $30,982 and wrongly ruled out the credit under the $24,680 limit. The correct $20,590 gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It said the head was ineligible for the Homestead Credit without any income test. The head is 85 and disabled, has $20,590 in household income and pays $7,200 in rent, which gives $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in household income ($30,982) instead of using wages plus Social Security ($20,590). The correct figure is under the limit and gives $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit, which requires counting the $10,392 survivor benefits. Homestead household income is $20,590 and the credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It said income was over the $24,680 limit by including the separate survivor benefits. Wages plus Social Security total $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
+us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used $30,982 as household income by adding the $10,392 survivor benefits. The correct base is $20,590, which qualifies for a $271.39 Homestead Credit."
+us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as pension income in homestead household income, getting $30,982. Homestead household income is $20,590 (wages plus Social Security), which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It never looked at the Homestead Credit, even though this elderly, disabled renter qualifies. With $20,590 in household income and $1,440 in rent-based property taxes, the credit is $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,It looked only for EITC-type refundable credits and missed the Homestead Credit. This 85-year-old disabled renter qualifies for $271.39.
+us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It counted both the Social Security and the separate $10,392 survivor benefits, pushing household income over the limit. Wages plus Social Security total $20,590, which gives $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It wrongly concluded that no refundable credit applies to an elderly filer without children and missed the Homestead Credit. That credit pays elderly or disabled renters with income under $24,680, here $271.39 on $20,590 of income."
+us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It said income was over the Homestead Credit limit, which only holds if the $10,392 survivor benefits are counted. Homestead household income is $20,590, which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It saw no sign of eligibility and missed that the Homestead Credit applies to an 85-year-old disabled renter paying $7,200 in rent. With $20,590 in household income the credit is $271.39."
+us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits, got $30,982 and compared that with a wrong $24,480 limit. Correct household income is $20,590, which qualifies for $271.39."
+us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit by including the separate survivor benefits. Wages plus Social Security total $20,590 and give a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Its total income figure counted the $10,392 survivor benefits on top of wages and Social Security. Homestead household income is $20,590, under the limit, which gives $271.39."
+us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It counted all three income streams against the Homestead Credit limit. The base is $20,590 (wages plus Social Security), which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
+us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It gave no analysis and missed the Homestead Credit, for which this elderly, disabled renter with $20,590 in household income qualifies at $271.39."
+us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in household income. The correct $20,590 is under the limit and gives $271.39."
+us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It included the separate $10,392 survivor benefits, putting household income at $30,982 instead of $20,590, and so wrongly ruled out a $271.39 Homestead Credit."
+us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly counted the other survivor benefits in homestead household income. Wages plus Social Security total $20,590, which gives 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39."
+us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Its $30,982 figure includes the $10,392 survivor benefits, and it compared that with a made-up limit of about $26,500. Correct household income is $20,590, which gives $271.39."
+us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It checked only the Wisconsin earned income credit and missed the Homestead Credit. This 85-year-old disabled renter with $20,590 in household income gets $271.39."
+us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there was nothing to score against the $271.39 Homestead Credit."
+us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a broad household income of $30,982 that includes the separate $10,392 survivor benefits. Homestead household income is $20,590, which qualifies for $271.39."
+us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,It wrongly assumed refundable credits require Wisconsin tax liability. The Homestead Credit is paid regardless of liability and comes to $271.39 for this renter.
+us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 other survivor benefits in household income, getting $30,982. The correct $20,590 is under the limit and gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It summed wages, Social Security and the separate $10,392 survivor benefits to get $30,982. Homestead household income is $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39."
+us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,It tied refundable credits to having qualifying children and missed the Homestead Credit. That credit is available to childless elderly or disabled renters and is $271.39 here.
us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP."
us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP."
us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band."
@@ -8511,46 +8724,47 @@ us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_elig
us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified the infant category and computed income at roughly 0.79 FPL against a ~200% FPL Florida infant limit, then reversed its own conclusion by imposing a Florida asset test and requiring Medicaid expansion or confirmed enrollment. MAGI-based children's categories carry no asset or resource test, non-expansion status restricts only the adult expansion group, and CHIP is residual coverage for children who fail Medicaid rather than a substitute for it — so the infant category grants eligibility, and the question asked for eligibility rather than enrollment."
us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed WIC status as disqualifying instead of computing eligibility from the supplied facts. The age-zero child satisfies WIC's child category, and the five-person household's income is below the 185%-of-poverty income limit, so the correct eligibility output is Yes."
us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an overstated projected standard deduction, incorrectly concluded taxable income and pre-credit tax were zero, and then subtracted an invented $284 of “unused” CTC. The correct computation produces income tax that uses $2,792.91 of the $6,600 CTC as a nonrefundable offset, while the remaining $3,807.09 is refundable; unused nonrefundable credits never make this output negative."
-us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It derived the exact components: EITC $8,231 plus ACTC of 15% × ($27,880 − $2,500) = $3,807, for $12,038. It then rounded the submitted value to $12,000, throwing away the precision of its own correct computation."
-us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It denied the EITC entirely by claiming $28k of income 'exceeds' a phase-out range that it said ends near $50,162. In fact $27,881 of earned income is on the plateau, so the full $8,231 applies. It also computed a refundable CTC of $3,877.50 but submitted an unsupported $2,500."
-us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own recomputation reached the correct $8,231 EITC + $3,807 ACTC = $12,038. It then submitted an unexplained $10,379 that matches none of its intermediate figures."
-us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $8,046 instead of the 2026 figure of $8,231. It first computed ACTC on the $27,705 net-SE base (0.9235 × $30,000) rather than on earned income after the half-SE-tax deduction ($27,881). It then replaced that $3,781 with an unsupported $4,416 for the submitted total."
-us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated $30,000 as falling in the EITC phase-out and cut the credit to about $5,035, but $27,881 of earned income is below the MFJ phase-out start, so the full $8,231 applies. It also computed ACTC on gross $30,000 ($4,125) instead of on $27,881 net of half SE tax ($3,807)."
-us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It phased down the EITC using a $26,511 threshold applied to gross $30,000, which gave about $7,469. The 2026 MFJ phase-out starts above the $27,881 of earned income, so the full $8,231 applies. It also used gross $30,000 for the ACTC ($4,125 instead of $3,807). It then submitted $7,594, which drops $4,000 from its own stated $11,594 total."
-us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It counted the ACTC at the full $5,100 cap even though it had just computed the 15% earned-income limit at $3,807, which is the binding amount. It also cut the EITC to about $3,015 through a phase-out that does not apply at $27,881 of earned income, where the plateau credit is $8,231."
-us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It computed the ACTC as 15% of gross $30,000 above $2,500 ($4,125). Earned income for self-employment is net earnings minus the deductible half of SE tax ($27,881), which yields $3,807. It also used an EITC maximum of $8,215 instead of the 2026 figure of $8,231."
-us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $3,000 refundable CTC ($1,000 per child, pre-TCJA law). In 2026 the refundable CTC is 15% of earned income over $2,500, capped at $1,700 per child, which gives $3,807 here. Its EITC of $8,026 is also below the 2026 maximum of $8,231."
-us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $3,807, computed on $27,881 of earned income, is correct. It overstated the 2026 three-child EITC maximum as $8,261 instead of $8,231, which leaves the total $30 high."
-us,scenario_109,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed the ACTC on gross $30,000 ($4,125) without subtracting the deductible half of SE tax, which leaves $27,881 of earned income and a $3,807 ACTC. It also overstated the 2026 EITC maximum as $8,290 instead of $8,231."
-us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a $1,000-per-child refundable CTC ($3,000) instead of the 2026 formula: 15% of ($27,881 − $2,500) = $3,807, under the $1,700-per-child cap. It also overstated the EITC maximum as $8,290 instead of $8,231."
-us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It reduced the EITC to $3,588 as if the household were deep in the phase-out, but $27,881 of earned income is on the three-child plateau, which pays $8,231. It also understated the refundable CTC as $1,674 instead of 15% × ($27,881 − $2,500) = $3,807."
-us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $1,000-per-child ACTC ($3,000) instead of the 2026 formula, 15% of earned income over $2,500 capped at $1,700 per child, which gives $3,807. It also estimated the EITC maximum at $8,250 instead of $8,231."
-us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took the ACTC as a flat $1,000 per child ($3,000). The 2026 refundable CTC is 15% of ($27,881 − $2,500) = $3,807, under the $1,700-per-child cap. Its EITC maximum of $8,240 is also above the 2026 figure of $8,231."
-us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no computation, and its $7,502 is below the $8,231 plateau EITC alone. That means it reduced the EITC for a household whose $27,881 of earned income is below the MFJ phase-out start, and it left out the $3,807 refundable CTC."
-us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $3,000 refundable CTC ($1,000 per child) instead of the 2026 ACTC formula: 15% of ($27,881 − $2,500) = $3,807, capped at $1,700 per child. It also understated the EITC maximum as $8,219 instead of $8,231."
-us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $11,046 total equals the 2025 three-child EITC maximum of $8,046 plus a $1,000-per-child ACTC of $3,000. The correct 2026 figures are an EITC of $8,231 and a refundable CTC of 15% × ($27,881 − $2,500) = $3,807."
-us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a refundable CTC of $1,000 per child ($3,000) instead of the 2026 formula: 15% of earned income over $2,500 = $3,807, below the $1,700-per-child cap. It also used an EITC maximum of $8,207 instead of $8,231."
-us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used $27,705 (0.9235 × $30,000) as earned income. For self-employment, earned income is net SE earnings minus the deductible half of SE tax: $30,000 − $2,119 = $27,881. That left its ACTC at $3,780.75 instead of $3,807. It also used an EITC maximum of $8,025.75 instead of the 2026 figure of $8,231."
-us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted the half-SE-tax deduction from the already-reduced $27,705 net-earnings base, which double-counts the reduction and gives $25,586. Earned income is $30,000 − $2,119 = $27,881, which yields a $3,807 ACTC instead of its $3,463. It also used an EITC maximum of $8,042 instead of $8,231."
-us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $13,400 matches a roughly $8,300 EITC plus the full $5,100 refundable CTC cap ($1,700 × 3). It ignored the ACTC earned-income phase-in, which limits the refundable CTC to 15% of ($27,881 − $2,500) = $3,807."
-us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned zero on the grounds that there were no wages. But $30,000 of self-employment income is earned income for both the EITC and the ACTC. That qualifies the household for the $8,231 plateau EITC and a $3,807 refundable CTC."
-us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It had the EITC right at $8,231 but applied a $1,000-per-child refundable CTC ($3,000). The 2026 ACTC is 15% of ($27,881 − $2,500) = $3,807, capped at $1,700 per child."
-us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but ran the ACTC formula on gross $30,000 ($4,125). SE earned income must be reduced by the deductible half of SE tax to $27,881, which gives $3,807."
-us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but based the ACTC on $27,705 (0.9235 × $30,000), which gave $3,781. Earned income for the credit is $30,000 minus the $2,119 deductible half of SE tax, i.e. $27,881, which yields $3,807."
-us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but computed the ACTC on gross $30,000 ($4,125). It did not subtract the deductible half of SE tax, which brings earned income to $27,881 and the ACTC to $3,807."
-us,scenario_109,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but applied 15% over $2,500 to gross $30,000 ($4,125). Earned income for self-employment is net of the deductible half of SE tax ($27,881), which gives an ACTC of $3,807."
-us,scenario_109,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $3,807.09 is exactly right. It overstated the 2026 three-child EITC maximum as $8,427 instead of $8,231, which leaves the total $196 high."
-us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It computed no refundable credits at all. Self-employment income of $30,000 is earned income, which qualifies this MFJ three-child family for the $8,231 plateau EITC and a $3,807 refundable CTC."
-us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA expired and used a $1,000-per-child ACTC with a $3,000 earnings threshold. For 2026 the refundable CTC is 15% of earned income over $2,500, capped at $1,700 per child, which gives $3,807 on $27,881. Its EITC maximum of $8,226 is also slightly below $8,231."
-us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It treated the 2026 CTC as $1,000 per child with a $3,000 earnings threshold, giving $3,000. The applicable ACTC is 15% × ($27,881 − $2,500) = $3,807 under a $1,700-per-child cap. It also used an EITC maximum of $8,247 instead of $8,231."
-us,scenario_109,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It had the EITC right at $8,231 but explicitly assumed TCJA expired, which led it to a $1,000-per-child refundable CTC and a $3,000 earnings threshold. The 2026 rules keep the $2,500 threshold, the 15% rate and a $1,700-per-child refundable cap, which gives $3,807."
-us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA ACTC parameters: $1,000 per child and a $3,000 threshold, capping the credit at $3,000. The 2026 refundable CTC is 15% of ($27,881 − $2,500) = $3,807. Its EITC maximum of $8,245 is also above $8,231."
-us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its ACTC of about $3,807 is correct. It overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231, which leaves the total $40 high."
-us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"It gave no explanation, and its $6,695 is below the $8,231 plateau EITC alone. That means it reduced the EITC as if $27,881 of earned income were in the MFJ phase-out, and it did not add the $3,807 refundable CTC."
-us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It assumed an MFJ phase-out start near $29,600 and cut the EITC to $3,588, but $27,881 of earned income is on the plateau and gets the full $8,231. It first set the ACTC at the full $5,100 cap, ignoring the 15% earned-income limit of $3,807, then submitted an unsupported $4,288 total."
-us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It claimed the 15% earnings test supports the full $5,100 refundable CTC. In fact 15% × ($27,881 − $2,500) = $3,807, and that binds. It also phased the EITC down from a nonexistent ~$24,000 MFJ threshold to $7,414, although $27,881 is on the plateau at $8,231. Its $12,013.75 submission is not the $12,514 its own explanation states."
-us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It phased the EITC down using an assumed $24,000–$25,500 MFJ threshold, then replaced that with an arbitrary $5,475. The household's $27,881 of earned income is below the MFJ phase-out start, so the full $8,231 applies. It also computed the ACTC on gross $30,000 ($4,125) instead of on $27,881 ($3,807)."
-us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated the CTC as nonrefundable and set the ACTC to zero. The household has no income tax liability and $27,881 of earned income, so it receives a refundable CTC of 15% × ($27,881 − $2,500) = $3,807. It also used the 2025 EITC maximum of $8,046 instead of the 2026 figure of $8,231."
+us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived EITC $8,231 plus ACTC $3,807 for a total of $12,038. It then rounded the submitted value to $12,000, which threw away $38.09 of a computed exact amount."
+us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model claimed that $28k of income was above the EITC phase-out, so it gave no EITC. In fact, $27,880.57 of earned income is on the MFJ three-child plateau and earns the full $8,231. It also computed an ACTC of $3,877.50 and then submitted an arbitrary $2,500 instead."
+us,scenario_109,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated $30,000 of self-employment income as producing no EITC and roughly zero refundable CTC. SE profit minus half of SE tax ($27,880.57) is earned income for both credits. It produces the full $8,231 EITC and an ACTC of $3,807.09."
+us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's own arithmetic reached the correct total: EITC $8,231 + ACTC $3,807 = $12,038. It then submitted an unsupported $10,379 that matches none of its intermediate figures."
+us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum ($8,046) instead of the 2026 value ($8,231). It computed the ACTC on $27,705 of net SE earnings rather than on $27,880.57 (profit minus half of SE tax). It then replaced its $3,781 ACTC with an unexplained $4,416 in the final sum."
+us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household in the EITC phase-out and cut the credit to about $5,035. Earned income of $27,880.57 is below the MFJ three-child phase-out start, so the full $8,231 applies. It also computed the ACTC on gross $30,000 ($4,125) instead of on earned income net of the half-SE-tax deduction ($3,807.09)."
+us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied an EITC phase-out starting near $26.5k, which is the wrong threshold. The MFJ three-child phase-out begins above $30k, so the full $8,231 applies. It computed the ACTC on gross $30,000 instead of $27,880.57. Finally, it dropped the thousands digit of its own $11,594 total and submitted $7,594."
+us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model cut the EITC to about $3,015 as if it were deep in the phase-out. At $27,880.57 of earned income, a three-child MFJ household is on the plateau at $8,231. It also set the ACTC to the $5,100 per-child cap even though its own 15%-of-earnings-over-$2,500 limit gave $3,807."
+us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model computed the ACTC as 15% of gross $30,000 minus $2,500 ($4,125). Earned income for the ACTC is SE profit reduced by the deductible half of SE tax ($27,880.57), which gives $3,807.09. It also used $8,215 for the 2026 EITC maximum instead of $8,231."
+us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model set the refundable CTC at $3,000 ($1,000 per child), as if the TCJA had expired. The 2026 CTC allows up to $1,700 refundable per child, limited to 15% of earnings over $2,500, which gives $3,807.09. It also understated the 2026 EITC maximum as $8,026 instead of $8,231."
+us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The ACTC of $3,807 is correct. The model overstated the 2026 three-child EITC maximum as $8,261 instead of $8,231, so its total is $30 too high."
+us,scenario_109,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model computed the ACTC on gross $30,000 ($4,125) instead of on earned income net of the half-SE-tax deduction ($27,880.57 → $3,807.09). It also overstated the 2026 EITC maximum as $8,290 instead of $8,231."
+us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000-per-child ACTC ($3,000) from expired pre-TCJA law. The 2026 refundable cap is $1,700 per child, and the binding 15% earned-income formula gives $3,807.09. It also overstated the EITC maximum as $8,290 instead of $8,231."
+us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model cut the EITC to $3,588 as if it were deep in the phase-out. The household's $27,880.57 of earned income is on the three-child MFJ plateau, so the EITC is $8,231. Its $1,674 refundable CTC also understates 15% × ($27,880.57 − $2,500) = $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a flat $3,000 ACTC ($1,000 per child) instead of the 2026 formula. That formula, 15% of earned income over $2,500 with a $1,700-per-child cap, gives $3,807.09. It also overstated the EITC maximum as $8,250 instead of $8,231."
+us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated the maximum ACTC as $1,000 per child ($3,000). Under 2026 law the cap is $1,700 per child, and the earned-income formula gives $3,807.09. It also set the EITC maximum at $8,240 instead of $8,231."
+us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no derivation. Its $7,502 is $4,536 below the correct total of the full $8,231 plateau EITC plus the $3,807.09 ACTC (15% of the $27,880.57 earned income above $2,500), so it badly under-counted at least one of the two credits."
+us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 ACTC ($1,000 per child) from expired-TCJA law instead of the 2026 formula, which yields $3,807.09. It also used $8,219 for the EITC maximum instead of $8,231."
+us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $11,046 total matches the 2025 EITC maximum ($8,046) plus a $1,000-per-child ACTC ($3,000). The correct figures are the 2026 maximum of $8,231 and an ACTC of 15% × ($27,880.57 − $2,500) = $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied a $1,000-per-child ACTC ($3,000) instead of the 2026 formula, which yields $3,807.09 under the $1,700-per-child cap. It also understated the EITC maximum as $8,207 instead of $8,231."
+us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model understated the 2026 EITC maximum as $8,025.75 instead of $8,231. It computed the ACTC on $27,705 (the 0.9235 SE-tax base) instead of on SE profit minus half of SE tax ($27,880.57), which gave $3,780.75 rather than $3,807.09."
+us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model reduced earned income twice. It applied the 0.9235 factor and also subtracted half of SE tax, getting $25,586 instead of $30,000 − $2,119.43 = $27,880.57. That cut the ACTC to $3,463 instead of $3,807.09. It also used $8,042 for the EITC maximum instead of $8,231."
+us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no computation and submitted $13,400, which overstates the total. The correct total is the $8,231 maximum EITC plus an ACTC limited to 15% of earned income over $2,500, or $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model refused to count self-employment income as earned income for the EITC and ACTC. SE profit minus half of SE tax ($27,880.57) is earned income under IRC §32(c)(2)(A)(ii), and it qualifies the household for an $8,231 EITC and a $3,807.09 ACTC."
+us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The EITC of $8,231 is correct. The model used a $1,000-per-child ACTC ($3,000) as if the TCJA had expired. The 2026 CTC has a $1,700-per-child refundable cap, so the earned-income formula governs and yields $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The ACTC was computed on gross $30,000 ($4,125) instead of on earned income reduced by the deductible half of SE tax ($27,880.57), which gives $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The EITC of $8,231 is correct. The model computed the ACTC on $27,705 (the 92.35% SE-tax base) instead of on SE profit minus the deductible half of SE tax ($27,880.57). That gave $3,781 instead of $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The ACTC of $4,125 uses gross $30,000 as earned income. It should use $30,000 minus the $2,119.43 half-SE-tax deduction, which yields $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The model computed the ACTC as 15% of gross SE income over $2,500 ($4,125) without first subtracting the deductible half of SE tax. The correct figure is 15% × ($27,880.57 − $2,500) = $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The ACTC of $3,807.09 is exactly right. The model overstated the 2026 three-child EITC maximum as $8,427 instead of $8,231, so its total is $196 too high."
+us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model computed no EITC or ACTC, although $27,880.57 of SE earned income and three qualifying children produce a full $8,231 EITC and a $3,807.09 ACTC."
+us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied expired-TCJA rules: a $1,000-per-child CTC and a $3,000 earnings threshold. For 2026, the CTC has a $1,700 refundable cap per child and a $2,500 threshold, which yields $3,807.09. It also used $8,226 for the EITC maximum instead of $8,231."
+us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model treated the 2026 CTC as $1,000 per child with a $3,000 threshold, capping the ACTC at $3,000. Current law sets a $1,700 refundable cap and a $2,500 threshold, so the ACTC is $3,807.09. It also overstated the EITC maximum as $8,247 instead of $8,231."
+us,scenario_109,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The EITC of $8,231 is correct. The model assumed the TCJA had expired, giving a $1,000-per-child CTC and a $3,000 threshold. The 2026 CTC keeps the $2,500 threshold and a $1,700-per-child refundable cap, so the ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09."
+us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped the ACTC at $3,000 ($1,000 per child, with a $3,000 threshold) under expired-TCJA rules instead of the 2026 $1,700 cap and $2,500 threshold, which gives $3,807.09. It also overstated the EITC maximum as $8,245 instead of $8,231."
+us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ACTC of about $3,807 is correct. The model overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231, so its total is $40 too high."
+us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The model gave no reasoning. Its $6,695 is $5,343 below the correct total of the full $8,231 plateau EITC plus a $3,807.09 ACTC, so it missed most of one credit or wrongly phased down the EITC."
+us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model assumed an EITC phase-out start near $29,600 and cut the credit to $3,588. Earned income of $27,880.57 is on the plateau, so the EITC is $8,231. It also ignored the 15%-of-earnings limit on the ACTC ($3,807.09), and its submitted $4,288 does not even match its own $8,688 sum."
+us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used a phase-out start of about $24,000 that is too low and phased the EITC down to $7,414. The MFJ three-child plateau extends beyond $27,880.57, so the full $8,231 applies. It also set the ACTC to the full $5,100 cap, ignoring the binding 15% × (earned income − $2,500) limit of $3,807.09."
+us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model applied an EITC phase-out starting at about $24,000–$25,500 and settled on an arbitrary $5,475. A three-child MFJ household at $27,880.57 of earned income gets the full $8,231. It also computed the ACTC on gross $30,000 ($4,125) instead of on income net of the half-SE-tax deduction ($3,807.09)."
+us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the CTC as entirely nonrefundable and set the ACTC to zero. With no tax liability and $27,880.57 of earned income, the refundable ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09. It also used the 2025 EITC maximum of $8,046 instead of the 2026 value of $8,231."
us,scenario_109,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility without running either qualifying pathway: SNAP receipt of $8,020.55 confers categorical free-meal status by itself, and the household's $30,000 income is 78% of the 2026 five-person federal poverty guideline, far below the 130% free-meal threshold. Its bare 0 is consistent with demanding an explicit school-enrollment fact instead of deriving two K-12 students from the stated ages of 9 and 6, and Florida has no universal free-meal program that would make the answer turn on state policy."
us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated school-meal eligibility as an unlisted fact to be zeroed out under the 'treat unlisted household facts as false' instruction, so it never applied the SNAP categorical qualifier or the 130% poverty-guideline income test. Ages 9 and 6 establish two children in K-12, the household's $8,020.55 SNAP benefit is a standalone categorical pathway to free meals, and $30,000 is 78% of the five-person poverty guideline — each route independently yields the FREE tier and $2,261.92 of annual support."
us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as offering Medicaid expansion to low-income adults and therefore used low MAGI as sufficient for eligibility. Florida has no expansion pathway for this age-30 head, and the head fits no other categorical pathway, producing medicaid_category NONE and ineligibility."
@@ -8564,47 +8778,48 @@ us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode
us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model set up 30000 × 0.9235 × 0.153 correctly but evaluated it as 4234 when the product is 4238.87, understating the tax by $4.87 — its number implies a net-earnings base of $27,673 instead of $27,705. It then compounded the confusion with an irrelevant Additional Medicare Tax digression and a stray discarded figure of 2793 before committing to the miscomputed 4234."
us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,other,False,"The model named the correct base (92.35% of $30,000) and the correct combined 15.3% Social Security and Medicare rate, but reported the product as $4,241 when $27,705 × 0.153 = $4,238.87, which rounds to $4,239. The $2.13 overstatement is a multiplication slip presented as a rounding, equivalent to using a net-earnings base of $27,719."
us,scenario_109,self_employment_tax,minimax-m3,llm_error,other,False,"The model computed every component correctly — $27,705 net earnings, 12.4% Social Security of $3,435.42, 2.9% Medicare of $803.45 — and stated the total as approximately $4,238, then submitted $4,233, contradicting its own arithmetic by $5. Its own component figures sum to $4,238.87, so the submitted value is a transcription error rather than a rule or base mistake."
-us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model first got about $625 a month using stale FY2025 figures: a $225 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It then cut the benefit to $314 a month by subtracting 'payroll/SE adjustments', but SNAP does not deduct self-employment tax from countable income. The correct result is $1,183 − $522 = $661 a month."
-us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"The inputs were right ($1,739 net income, $1,183 maximum), but the model left the 30% contribution at $521.70. SNAP rounds it up to the next whole dollar ($522) under 7 CFR 273.10(e)(2)(ii)(A). That makes the monthly benefit $661 rather than $661.30 and the annual total $7,932, not $7,936."
-us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model subtracted half of the self-employment tax from gross income, but SNAP counts self-employment income before income tax adjustments. It also used a $1,316 maximum instead of $1,183. Its $244 monthly benefit does not follow from the maximum-minus-30%-of-net formula, which gives $1,183 − $522 = $661."
-us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The method was right, but the model used a $1,613 maximum allotment for five, well above the actual FY2026 figure of $1,183. It also used a $240 standard deduction instead of $261. The result was a $1,085 monthly benefit instead of $661."
-us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model cut income by 7.65% for self-employment tax and assumed a rent-based shelter deduction lowered net income. SNAP does not deduct self-employment tax. Rent of $800 is below half of adjusted income ($869.50), so the excess shelter deduction is zero. It also used the FY2025 $1,158 maximum instead of $1,183."
-us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model claimed an excess shelter deduction for the $9,600 rent. None exists: $800 of rent is below half of the $1,739 adjusted income. Even so, its $424 monthly result is lower than the correct $661, so it does not follow the $1,183 − 30% of $1,739 formula at all."
-us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the FY2026 maximum for five at $1,017 instead of the actual $1,183. It also used the FY2025 four-person standard deduction ($228) instead of the $261 five-person figure. The result was $485 a month instead of $661."
-us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model counted six people instead of five and cut income by 7.65% for self-employment tax. It then ruled the household ineligible even though its own numbers passed both tests: $2,309 of gross income under a $4,590 limit and net income under a $3,532 limit. The household passes both tests and gets $661 a month."
-us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the right $1,739 net income and $1,183 maximum. It did not round the $521.70 contribution up to $522 as 7 CFR 273.10(e)(2)(ii)(A) requires, so it annualized $661.30 into $7,936 instead of $661 × 12 = $7,932."
-us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $350 standard deduction instead of the FY2026 five-person figure of $261. That understated net income ($1,650 instead of $1,739). Combined with a $1,185 maximum instead of $1,183, it gave $690 a month instead of $661."
-us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $232 standard deduction instead of $261, which overstated net income ($1,768 instead of $1,739). It also used a $1,184 maximum instead of $1,183, giving $654 a month instead of $661."
-us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used stale figures: a $244 standard deduction instead of $261 and the FY2025 $1,158 maximum instead of the FY2026 $1,183. The result was $631 a month instead of $661."
-us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The $1,183 maximum was right, but the model used a $250 standard deduction instead of $261. Net income came out at $1,750 instead of $1,739 and the contribution at $525 instead of $522, giving $658 a month instead of $661."
-us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model's $1,750 net income implies a $250 standard deduction instead of $261. Its $695 monthly benefit implies a maximum of about $1,220 instead of $1,183. Both errors are parameter mistakes on top of the correct $2,500 − $500 − $261 method."
-us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $765 monthly benefit implies net income of about $1,393, roughly $346 below the correct $1,739. That matches adding an excess shelter deduction or utility allowance. Neither applies: rent of $800 is below half of adjusted income and no utility costs are listed."
-us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly applied an excess shelter deduction, which pulled net income down to about $1,323 and the benefit up to $786 a month. Shelter costs of $800 are below half of the $1,739 adjusted income ($869.50), and no utilities are listed, so the shelter deduction is zero and the benefit is $661."
-us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's $1,750 net income implies a $250 standard deduction instead of $261. Its $665 monthly benefit implies a maximum of about $1,190 instead of $1,183. The correct figures give $1,183 − $522 = $661."
-us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model answered $0 with no calculation. The household passes the gross income test ($2,500 a month against a $3,137.50 guideline), the net income test ($1,739) and the asset test ($900). The benefit is $1,183 − $522 = $661 a month."
-us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated the $800 rent as producing a deduction, and its $752.50 monthly benefit implies net income of about $1,435. Rent is only deductible above half of adjusted income ($869.50), so this household's shelter deduction is zero. Net income stays at $1,739."
-us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $732 monthly benefit implies net income of about $1,503, around $236 below the correct $1,739. That matches applying a shelter deduction the household does not qualify for, since $800 of rent is below half of adjusted income ($869.50)."
-us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $675 monthly benefit is $14 above the correct $661. That requires either overstating the $1,183 FY2026 maximum (about $1,197) or deducting about $46 more than the $761 allowed. The $800 rent it cites produces no shelter deduction."
-us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used a $236 standard deduction instead of $261 and a $1,155 maximum instead of $1,183. It also did not round the 30% contribution up to a whole dollar, which gave $625.80 a month instead of $661."
-us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"The model used a $258 standard deduction instead of $261 and a $1,180 maximum instead of $1,183. That gave about $657 a month instead of $661."
-us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model claimed very low net income and high shelter costs, but those would raise the benefit above $661, not lower it to about $485. Its number matches subtracting 30% of income from the FY2025 $1,158 maximum without taking the $500 earned income deduction. The correct figure is $1,183 − 30% of $1,739."
-us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model assumed no take-up and returned $0, even though the prompt says to assume take-up whenever it is required. It also skipped the income calculation, which shows the household eligible for $661 a month."
-us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $659 monthly benefit is exactly the $1,183 maximum minus 30% of $1,746, which comes from using the FY2025 $254 standard deduction instead of the FY2026 $261. The correct net income is $1,739 and the benefit is $661."
-us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $659 monthly benefit is exactly $1,183 minus 30% of $1,746, which uses the FY2025 $254 standard deduction instead of the FY2026 $261. The correct net income is $1,739 and the benefit is $661."
-us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"The deductions and maximum were right, but the model kept the contribution at $521.70. 7 CFR 273.10(e)(2)(ii)(A) rounds it up to $522, so the benefit is $661 a month and $7,932 a year, not $661.30 and $7,935.60."
-us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $8,045 works out to about $670.42 a month. That is not a whole-dollar SNAP amount, and it is $9.42 above the correct $1,183 − $522 = $661. It reflects an overstated maximum or understated standard deduction plus no rounding of the contribution."
-us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's $1,739 net income and $1,183 maximum were correct. It did not round the $521.70 contribution up to $522, so it reported $661.30 a month ($7,935.60 a year) instead of $661 ($7,932)."
-us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"The model said income and assets exceed SNAP limits. Gross income of $2,500 a month is well below 130% of the $3,137.50 guideline, net income of $1,739 is below 100%, and $900 in assets is below the asset limit. The household is eligible for $661 a month."
-us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected a $1,213 maximum instead of the published FY2026 $1,183. It also used a standard deduction that gave $1,744 of net income instead of $1,739. The result was $690 a month instead of $661."
-us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"The model's net income ($1,740) was essentially right, but it used a maximum of about $1,190 instead of $1,183. That gave $668 a month instead of $661."
-us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"The model used FY2025 figures: a $254 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It also annualized without rounding the contribution, giving $7,610.40 instead of $7,932."
-us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,189 maximum instead of $1,183 and the FY2025 $254 standard deduction. It also made an arithmetic slip: $2,000 − $254 is $1,746, not the $1,740 it wrote. The result was $667 a month instead of $661."
-us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"The model used a standard deduction of about $250 instead of $261, which put net income at $1,750. It also used a maximum of about $1,190 instead of $1,183. That gave $665 a month instead of $661."
-us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $300 monthly standard deduction instead of $261, which put net income at $1,700. It also projected a $1,220 maximum instead of $1,183. That gave $710 a month instead of $661."
-us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used a $1,151 maximum instead of $1,183 and first said net income was about $0. Its final $489 benefit implies a contribution of about $662, or net income near $2,207. That matches skipping the $500 earned income deduction; the correct net income is $1,739."
-us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"The model's explanation added a Florida utility allowance of about $400 even though no utility costs are listed, and used a $306 standard deduction instead of $261. That created a $356 shelter deduction and a $9,400 total. The submitted $7,920 contradicts that explanation and is still $12 below the correct $661 × 12."
-us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model awarded the full maximum allotment as if net income were zero. It skipped subtracting the 30% contribution ($522) from $1,739 of net income. It also used a $962 maximum instead of the FY2026 five-person $1,183."
-us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model used a four-person household when there are five people: two adults and three children. It also treated net income as zero, but after the $500 earned income and $261 standard deductions it is $1,739. Its $998 figure is the four-person maximum, not the five-person $1,183."
+us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a $225 standard deduction and a $1,158 maximum benefit instead of the FY2026 figures of $261 and $1,183. It then cut its own ~$625/month result to $3,768 a year by subtracting payroll/SE-tax adjustments. SNAP counts gross self-employment receipts minus business costs and never deducts SE tax, so the correct benefit is $1,183 − $522 = $661/month."
+us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"Its deductions, net income ($1,739) and $1,183 maximum are all right. However, it did not round the 30% contribution ($521.70) up to the next whole dollar ($522), as SNAP requires. That left $661.30/month instead of $661, and $7,936 instead of $7,932."
+us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It took half of SE tax off gross income, which SNAP does not do; SNAP counts the full $2,500/month of self-employment receipts. It also used a $1,316 maximum benefit instead of $1,183. Its final $244/month doesn't follow from its own numbers; the correct result is $1,183 − 30% of $1,739 net income (rounded up to $522) = $661."
+us,scenario_109,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It denied SNAP without doing any calculation. The household's $2,500 monthly gross income is under the gross income limit, and its $900 in assets passes the asset test. Net income is $1,739, so the benefit is $1,183 − $522 = $661/month."
+us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It used $1,613 as the five-person maximum benefit; the FY2026 figure is $1,183. It also used a $240 standard deduction instead of $261. With the correct figures, net income is $1,739 and the benefit is $661/month, not $1,085."
+us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It cut self-employment income by 7.65% for SE tax, which SNAP does not do. It used a $217 standard deduction instead of $261. It also applied a shelter deduction even though the $800 rent is below half of adjusted income ($869.50). Its $11,436 answer ($953/month) doesn't match its own $717/month result; the correct benefit is $1,183 − $522 = $661."
+us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction from the $9,600 rent. None applies, because $800/month is below half of the $1,739 adjusted income. Its $424/month figure doesn't follow from the $1,183 maximum minus the $522 contribution, which gives $661/month."
+us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took the FY2025 maximum benefit for five to be $967 and projected $1,017 for FY2026; the actual FY2026 figure is $1,183. It also used a $228 standard deduction instead of $261. Those two errors pushed the benefit down to $485/month instead of $661."
+us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six people; the household has five. It also cut SE income by 7.65%. It then concluded that income fails the limits even though its own figures (~$2,309 gross against a ~$4,590 limit) pass. The five-person household passes every test and gets $661/month."
+us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its net income ($1,739) and $1,183 maximum are correct. However, it did not round the 30% contribution ($521.70) up to $522, so it got $661.30/month instead of $661 and $7,936 instead of $7,932."
+us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person figure of $261, which understated net income at $1,650 instead of $1,739. It also used a $1,185 maximum instead of $1,183. Together these inflated the benefit to $690/month instead of $661."
+us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction instead of $261 and a $1,184 maximum instead of $1,183. That overstated net income ($1,768 instead of $1,739) and gave $654/month instead of $661."
+us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used outdated figures: a $244 standard deduction and a $1,158 maximum, instead of the FY2026 values of $261 and $1,183. That gave $631/month instead of $1,183 − $522 = $661."
+us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It had the right $1,183 maximum but used a $250 standard deduction instead of the FY2026 five-person figure of $261. That raised net income to $1,750 and the contribution to $525, giving $658/month instead of $661."
+us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its $1,750 net income reflects a ~$250 standard deduction instead of $261. Its $695/month benefit implies a maximum of about $1,220 instead of $1,183. The correct result is $1,183 − $522 = $661/month."
+us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $765/month is $104 too high. It implies net income of about $1,393, as though a ~$346 shelter deduction applied. No excess shelter deduction exists, because $800 rent is below half of the $1,739 adjusted income, so the benefit is $661/month."
+us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly subtracted an excess shelter deduction. That deduction is zero here: $800 rent minus half of $1,739 adjusted income ($869.50) is negative, and no utility costs are listed. Its $786/month should be $661."
+us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a ~$250 standard deduction instead of $261, giving $1,750 net income. Its $665/month implies a maximum of about $1,190 instead of $1,183. The FY2026 figures give $661/month."
+us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It answered $0 without doing any calculation. The five-person household with $2,500/month in earnings passes the gross, net and asset tests. Net income of $1,739 gives $1,183 − $522 = $661/month."
+us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It treated the $800 rent as a deduction. SNAP only deducts shelter costs above half of adjusted income, and $800 is below $869.50, so the shelter deduction is zero. Its $752.50/month should be $661."
+us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $732/month implies net income of about $1,503, which is $236 below the correct $1,739. That gap amounts to a deduction the household doesn't have; only the $500 earned income deduction and the $261 standard deduction apply, with no excess shelter. The correct benefit is $661/month."
+us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $675/month is $14 too high, which reflects an inflated maximum or understated net income. The FY2026 figures ($261 standard deduction, $1,183 maximum) give net income of $1,739, a $522 contribution and $661/month."
+us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"It used FY2025-era figures, a $236 standard deduction and a $1,155 maximum, instead of the FY2026 values of $261 and $1,183. It also skipped the whole-dollar round-up of the contribution. That produced $625.80/month instead of $661."
+us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $258 standard deduction instead of $261 and a $1,180 maximum instead of $1,183. That gave ~$657/month, which it then rounded loosely to $7,880 a year, instead of $661/month and $7,932."
+us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It claimed 'very low net income and high shelter costs,' but net income is $1,739 and the $800 rent produces no excess shelter deduction. Its $484.67/month is not derived from the $1,183 maximum minus the $522 contribution, which gives $661."
+us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed no take-up and answered $0, even though the prompt says to assume program take-up. The household is eligible, with $1,739 net income and a benefit of $661/month."
+us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $659/month matches using the FY2025 five-person standard deduction ($254) with the FY2026 $1,183 maximum. That gives $1,746 net income and a $524 contribution. The FY2026 standard deduction is $261, which gives $661/month."
+us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $659/month matches using the outdated FY2025 $254 standard deduction instead of the FY2026 $261. That overstated net income at $1,746 and the contribution at $524, so the result is $2/month below the correct $661."
+us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Its parameters are correct ($261 standard deduction, $1,183 maximum, $1,739 net income). However, it did not round the $521.70 contribution up to $522, as SNAP requires, so it reported 12 × $661.30 = $7,935.60 instead of 12 × $661 = $7,932."
+us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $8,045 works out to about $670.42/month, roughly $9 too high. That reflects an inflated maximum or an understated standard deduction. The FY2026 figures give $1,183 − $522 = $661/month."
+us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got $1,739 net income and the $1,183 maximum right. However, it left the 30% contribution at $521.70 instead of rounding it up to $522, so it reported $661.30/month ($7,935.60) instead of $661 ($7,932)."
+us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It claimed income and assets exceed SNAP limits. In fact, $2,500/month is well under the five-person gross limit, and $900 in assets is far below the asset limit. The household qualifies for $661/month."
+us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected the five-person maximum at ~$1,213 instead of the actual FY2026 figure of $1,183. That inflated the benefit to $690/month instead of $661."
+us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a ~$1,190 maximum instead of $1,183. Its other steps were right, so the error is the overstated maximum, which gave $668/month instead of $661."
+us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"It used the FY2025 $254 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It also applied the 30% contribution to annual income without the monthly round-up, which gave $7,610.40 instead of $7,932."
+us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used the FY2025 $254 standard deduction instead of $261. It then made an arithmetic slip, giving $1,740 instead of $1,746. It also used a $1,189 maximum instead of $1,183, which pushed the benefit to $667/month instead of $661."
+us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It used a ~$250 standard deduction instead of $261 and a ~$1,190 maximum instead of $1,183. That gave $665/month instead of $661."
+us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $300/month standard deduction instead of the FY2026 five-person $261 and a $1,220 maximum instead of $1,183. That understated net income ($1,700 instead of $1,739) and gave $710/month instead of $661."
+us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"It treated rent as producing a capped shelter deduction and said net income was about $0. In fact, the $800 rent is below half of adjusted income, so there is no shelter deduction and net income is $1,739. It also used a $1,151 maximum instead of $1,183. Its $489/month should be $661."
+us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"Its reasoning added a ~$400 Florida standard utility allowance, even though no utility expenses were listed, and used a $306 standard deduction instead of $261. That created a $356 shelter deduction that doesn't exist. Its submitted $7,920 ($660/month) also contradicts its own stated $9,400. With no shelter deduction and the contribution rounded up to $522, the benefit is $661/month."
+us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It applied the earned income deduction at the gross income test and then assumed the maximum benefit, ignoring the 30% net income contribution. It also used a $962 maximum instead of $1,183. Net income of $1,739 reduces the benefit by $522, to $661/month."
+us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It used a four-person maximum ($998); the household has five people, and the five-person maximum is $1,183. It also treated net income as zero. Net income is actually $1,739 after the $500 earned income deduction and $261 standard deduction, so the benefit is $661/month."
us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base."
us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis."
us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold."
@@ -8618,131 +8833,134 @@ us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligib
us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category."
us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false."
us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own computation reached $25,840.76 (AGI $169,943 including estate income, $16,100 standard deduction, 15% on $40,884), yet it submitted $21,120, a figure no step in its reasoning produces. It also included the $6,323 estate income that the reference leaves out of AGI, and it skipped the $387 nonitemizer charitable, $198.35 IRA and $1,264.59 QBI deductions that the reference applies."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction. The reference's AGI of $163,421.64 leaves out estate income and nets a $198.35 IRA deduction. The model also took no QBI deduction, versus the reference's $1,264.59. Its $16,100 standard deduction and $387 nonitemizer charitable deduction match the reference, so the whole $1,850.56 gap comes from how estate income, the IRA and QBI were treated."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-OBBBA $15,000 standard deduction instead of the 2026 figure of $16,100 and included the $6,323 estate income. It then subtracted an arbitrary $1,300 of nonrefundable credits from its own $27,137, although this childless filer has no such credits. The reference instead applies $16,100 plus the $387 charitable and $1,264.59 QBI deductions to an AGI of $163,421.64."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction, whereas the reference AGI of $163,421.64 excludes estate income. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. On top of that, it used guessed brackets (12,150/49,400/105,450) instead of the 2026 thresholds of 12,400/50,400/105,700."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,745 after the IRA deduction) and omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. The reference's AGI of $163,421.64 excludes estate income and its taxable income is $145,669.58. The model also put the top of the 22% bracket at $107,500 instead of the 2026 figure of $105,700."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its components (about $20,000 of ordinary tax plus $6,133 of 15% preferential tax) add up to roughly $26,133, but it stacked the preferential tax again on a total that already included it and submitted $30,150. It also included the $6,323 estate income and omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions that the reference applies."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took a $10,400 qualified overtime deduction by assuming 10 overtime hours a week from the 50-hour schedule. No overtime premium is listed, the reference applies no overtime deduction, and the model also ignored the phase-down above $150,000 MAGI. It also included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions; these errors partly cancelled and left it $533.42 below the reference."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,750 standard deduction and brackets it inflated itself (12,150/49,200/104,900) instead of the 2026 $16,100 deduction and 12,400/50,400/105,700 brackets. It included the $6,323 estate income that the reference keeps out of its $163,421.64 AGI, and it omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It computed $26,044.70 by itemizing with a $5,331 2%-floor miscellaneous deduction for unreimbursed employee expenses, a deduction that OBBBA permanently eliminated. It then cut the result to $20,200 with no supporting calculation. The reference applies the $16,100 standard deduction plus the $387 charitable and $1,264.59 QBI deductions to an AGI of $163,421.64."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction, whereas the reference AGI of $163,421.64 excludes estate income and nets a $198.35 IRA deduction. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. As a result, $7,259 of its ordinary income falls in the 24% bracket, while the reference's $104,785.81 of ordinary income stays entirely within 22%."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used the outdated $15,000 standard deduction instead of the 2026 figure of $16,100. It also overstated ordinary tax at $20,174 on $113,861; the 2026 brackets give $19,924.64. It included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions that the reference applies."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a personal exemption, a $6,800 standard deduction, 2%-floor miscellaneous deductions and a 28% bracket. OBBBA made the TCJA rates, the $16,100 standard deduction and the repeal of exemptions permanent. It also reduced wages by ESI premiums the reference does not subtract, and its $32,490 ordinary tax on about $101,000 of ordinary income is far higher than any 2026 schedule produces."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: an $8,650 standard deduction, a $5,450 personal exemption, and itemized deductions that included 2%-floor employee expenses. OBBBA permanently keeps the $16,100 standard deduction and the repeal of exemptions for 2026. It also subtracted the $6,889 ESI premiums from wages, whereas the reference's employment income is $95,755.10, net of only the 401(k) deferral."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. The reference's AGI of $163,421.64 excludes estate income, and its taxable income of $145,669.58 reflects all three deductions. The model's taxable income was $8,173 higher, which pushed $7,259 of ordinary income into the 24% bracket."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and claimed a $5,050 personal exemption and $5,331 of 2%-floor miscellaneous employee expenses, both of which OBBBA eliminated permanently. The reference uses the $16,100 standard deduction, the $387 nonitemizer charitable deduction and the $1,264.59 QBI deduction. The model also included the $6,323 estate income that the reference leaves out of AGI."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $19,688 matches the tax on ordinary income alone, leaving the $40,884 of qualified dividends and long-term gains untaxed. Ordinary taxable income is well above the $49,450 0% threshold, so all of that preferential income is taxed at 15% ($6,132.56 in the reference)."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used itemized deductions and personal exemptions under pre-TCJA brackets. OBBBA permanently kept the $16,100 standard deduction, the TCJA brackets and the repeal of exemptions. It also subtracted pre-tax health premiums from wages, whereas the reference's employment income is $95,755.10, net of only the 401(k) deferral."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: a $5,050 personal exemption, 2%-floor unreimbursed employee expenses and pre-TCJA brackets. The permanent OBBBA rules for 2026 instead give a $16,100 standard deduction and 10/12/22/24% brackets. It also included the $6,323 estate income that the reference leaves out of its $163,421.64 AGI."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $19,679 approximates the tax on ordinary income alone and leaves out the 15% tax on $40,884 of qualified dividends and long-term gains ($6,132.56 in the reference). That 15% rate applies because ordinary taxable income is above the $49,450 0% threshold."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $6,889 employer-sponsored insurance premiums to get an AGI of $163,054. The reference keeps employment income at $95,755.10, net of only the 401(k) deferral, and instead excludes the $6,323 estate income. Its $22,415 is $1,482 below the reference, so it also took deductions beyond the reference's $16,100 standard, $387 charitable and $1,264.59 QBI amounts."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages to get an AGI of $163,054. The reference instead keeps employment income at $95,755.10 and excludes the $6,323 estate income. Its $22,765 is $1,132 short of the reference, so it also took deductions beyond the reference's $16,100 standard, $387 charitable and $1,264.59 QBI amounts."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,350 standard deduction instead of the 2026 figure of $16,100 and included the $6,323 estate income, giving taxable income of $154,593 against the reference's $145,669.58. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions."
-us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,400 standard deduction and the 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 $16,100 deduction and 12,400/50,400/105,700 brackets. It also included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions that the reference applies."
-us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $40,884 of qualified dividends and long-term gains at 0%, even though its $112,959 of ordinary taxable income is already above the $49,450 0% threshold. All of that income belongs in the 15% bracket ($6,132.56 in the reference). It also included the $6,323 estate income and skipped the $387 charitable and $1,264.59 QBI deductions."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its own reasoning says federal tax stays positive for this high-income filer, yet it submitted $0. The reference taxes $104,785.81 of ordinary income ($17,764.88) and $40,883.77 of preferential income at 15% ($6,132.56)."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation and applied unspecified itemized medical and charity assumptions to reach $16,811, which is $7,086 below the reference. Itemized deductions fall short of the $16,100 standard deduction, and the reference's total is $17,764.88 of ordinary tax plus $6,132.56 of 15% tax on the $40,883.77 of preferential income."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. The reference's AGI of $163,421.64 excludes estate income, and its taxable income is $145,669.58. This left $7,259 of the model's ordinary income in the 24% bracket, while the reference's ordinary income stays entirely within 22%."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI and took no QBI deduction, whereas the reference's AGI of $163,421.64 excludes estate income and its deductions include $1,264.59 of QBI and the $198.35 IRA deduction. Its $387 nonitemizer charitable deduction matches the reference."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages and claimed a $9,000 qualified overtime deduction based on an overtime premium that was never listed. The reference keeps employment income at $95,755.10 and applies no overtime deduction. The model's $163,054 AGI also includes the $6,323 estate income that the reference excludes, and it omitted the $1,264.59 QBI deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It claimed an $8,400 qualified overtime deduction by inferring an overtime premium from the 50-hour schedule. No overtime premium is listed, and the reference applies no overtime deduction. It also included the $6,323 estate income in AGI and omitted the $387 charitable and $1,264.59 QBI deductions; these errors partly cancelled and left it $49.86 below the reference."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $6,889 ESI premiums from wages to reach an AGI of $163,054. The reference keeps employment income at $95,755.10 and instead leaves the $6,323 estate income out of AGI. The model also omitted the $198.35 IRA and $1,264.59 QBI deductions, so its taxable income is $897 above the reference's."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $24,500 with no computation and relied on 'standard nonrefundable credits' that do not exist for this childless filer. The reference total is $17,764.88 of ordinary tax plus $6,132.56 of preferential-rate tax, with no credits."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired: it itemized with 2%-floor employee expenses, took a $5,300 personal exemption and used restored pre-TCJA brackets. OBBBA permanently keeps the $16,100 standard deduction, the repeal of exemptions and the 10/12/22/24% brackets for 2026. It also included the $6,323 estate income that the reference excludes."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied the law that would have followed a TCJA expiration: 10/15/25% brackets, a $5,300 personal exemption and $5,331 of 2%-floor employee expenses. The permanent OBBBA rules for 2026 instead give a $16,100 standard deduction and brackets that keep the reference's $104,785.81 of ordinary income within 22%. It also included the $6,323 estate income that the reference excludes."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: a $5,450 personal exemption, 10/15/25% brackets and $5,331 of 2%-floor employee expenses. OBBBA made the $16,100 standard deduction, the TCJA brackets and the repeal of exemptions permanent for 2026. It also included the $6,323 estate income that the reference leaves out of its $163,421.64 AGI."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and used a $5,387 personal exemption, 10/15/25% brackets and a $5,335 miscellaneous deduction subject to the 2% floor. OBBBA permanently eliminated all of these for 2026, and the reference instead uses the $16,100 standard deduction plus the $387 charitable and $1,264.59 QBI deductions. The model also included the $6,323 estate income that the reference excludes."
-us,scenario_110,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It itemized $21,146 by including $5,331 of 2%-floor unreimbursed employee expenses, a miscellaneous deduction that OBBBA permanently eliminated. Without it, itemized deductions fall below the $16,100 standard deduction the reference uses. It also included the $6,323 estate income and omitted the $1,264.59 QBI deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed an $8,425.50 qualified overtime deduction from an estimated $10,400 premium that is not among the listed inputs; the reference applies no overtime deduction. It also included the $6,323 estate income in taxable income and omitted the $1,264.59 QBI deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed without any computation that PolicyEngine returned $0. The reference taxes $145,669.58 of taxable income, $104,785.81 of it at ordinary rates and $40,883.77 at 15%, for $23,897.44."
-us,scenario_110,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took no IRA or QBI deduction. The reference's AGI of $163,421.64 excludes estate income and nets a $198.35 IRA deduction, and the reference also takes $1,264.59 of QBI. The model's $16,100 standard deduction and $387 nonitemizer charitable deduction match the reference."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Its own reasoning totals $26,973, yet it submitted $43,879.20, a figure no step produces. The computation also skipped the $4,245 401(k) exclusion and used guessed brackets and a $15,750 standard deduction instead of the 2026 figure of $16,100."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It understated AGI at $113,617 by dropping about $50,000 of the listed interest, dividend and capital gain income; the reference AGI is $163,421.64. It then applied a $297.87 QBI deduction instead of the reference's $1,264.59 and subtracted $194.77 of nonrefundable credits for a filer who has none."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached $25,840.76, but it submitted $21,120, a number with no support in that reasoning. Its derivation also left out the 2026 non-itemizer charitable cash deduction ($387), which cuts about $93 of 24%-bracket tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a $15,000 standard deduction instead of OBBBA's $16,100. It deducted the $198 traditional IRA contribution, which is fully phased out for an active plan participant with about $170k of MAGI. It said charitable gifts cannot reduce tax, missing the $387 non-itemizer cash deduction. It then subtracted an invented $1,300 of nonrefundable credits from its $27,137."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed the tax and submitted a $0 placeholder, even though its own reasoning said the liability is substantial. The correct steps give $19,615 of ordinary-rate tax on $112,571.70 plus $6,133 of 15% tax on $40,884 of preferential income."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used guessed brackets (12,150/49,400/105,450) instead of the OBBBA 2026 single brackets (12,400/50,400/105,700), which overstated ordinary tax. It also skipped the $387 non-itemizer charitable cash deduction, so its taxable income was $153,843 instead of $153,455.47."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It allowed the $198 traditional IRA deduction, which is fully phased out for an active 401(k) participant with $169,943 of MAGI. It omitted the $387 non-itemizer charitable cash deduction and set the top of the 22% bracket at $107,500 instead of $105,700. Its total landed near the reference only because these errors partly cancelled."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It added the $6,133 preferential-rate tax on top of a total that already included it, ending at $30,150 even though its components (about $20,000 ordinary plus $6,133) add to about $26,150. It also allowed the phased-out $198 IRA deduction and missed the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It made up a $10,400 qualified overtime deduction from the hourly rate and weekly hours, but no overtime premium was given and unlisted amounts are 0. It also allowed the phased-out $198 IRA deduction and missed the $387 non-itemizer charitable deduction. Together these understated taxable income by about $10,000."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and its own inflation-guessed brackets instead of OBBBA's 2026 figures ($16,100 deduction; brackets at 12,400/50,400/105,700). It also left out the $387 non-itemizer charitable cash deduction, which together overstated ordinary-rate tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It itemized using a $5,331 deduction for unreimbursed employee business expenses above a 2% floor, but OBBBA permanently eliminated that deduction. It applied 2025 brackets instead of 2026 ones. It then cut its own $26,044.70 result to $20,200 for no stated reason."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $16,100 standard deduction, the 2026 brackets and the 15% preferential tax right, but missed the 2026 non-itemizer charitable cash deduction. Deducting the $387 of cash gifts lowers ordinary taxable income to $112,571.70 and the tax by about $93."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the old $15,000 standard deduction instead of OBBBA's $16,100 for 2026 and deducted the $198 IRA contribution, which phases out at this income. It also missed the $387 non-itemizer charitable deduction and overstated ordinary tax at $20,174."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a $4,850 personal exemption, the 2% miscellaneous deduction and old brackets that run up to 28%. It also subtracted the $6,889 of employer-paid premiums from wages, even though wages are given gross and only the 401(k) deferral reduces them. Its $32,490 ordinary tax on about $101k is also miscalculated."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules (a $5,450 personal exemption, an $8,650 standard deduction, and itemizing that includes miscellaneous deductions), but OBBBA made the TCJA structure permanent. It also wrongly took the $6,889 of employer premiums out of wages and allowed the phased-out $198 IRA deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the correct $16,100 standard deduction and 2026 brackets, but missed the non-itemizer charitable cash deduction ($387) that OBBBA added starting in 2026. This left taxable income $387 too high and the tax about $93 too high."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, taking a $5,050 personal exemption and a $5,331 deduction for 2%-floor employee business expenses, and taxing ordinary income at pre-TCJA rates. Under OBBBA the correct calculation is the $16,100 standard deduction plus the $387 charitable deduction, with the 10/12/22/24% brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It left out the $6,323 of estate income. Its $19,688 is about the ordinary-rate tax alone ($19,615), with no 15% tax on the $40,884 of qualified dividends and LTCG, which adds $6,132.56."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA had expired, using personal exemptions and itemized deductions, even though OBBBA made the TCJA rate and deduction structure permanent. It also subtracted pre-tax health premiums from wages, even though only the $4,245 traditional 401(k) deferral reduces reported wages."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied rules from a TCJA expiration that did not happen: a $5,050 personal exemption, employee business expenses above a 2% AGI floor, and pre-TCJA brackets that gave $20,928 of ordinary tax. The OBBBA approach is the $16,100 standard deduction plus the $387 non-itemizer charitable deduction, with 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $19,679 is about the ordinary-rate tax alone ($19,615 on $112,571.70) and leaves out the $6,132.56 of 15% tax on the $40,884 of qualified dividends and long-term gains."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took the $6,889 of employer-sponsored premiums out of wages, giving $163,054 of AGI, but only the $4,245 traditional 401(k) deferral reduces wages, so AGI is $169,942.94. That understated taxable income and the ordinary-rate tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It began from $163,054 of AGI after subtracting $6,889 of employer premiums from wages, which the facts do not support. Only the 401(k) deferral reduces wages, so the correct AGI is $169,942.94."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of OBBBA's $16,100 for 2026, used off-cycle brackets, and left out the $387 non-itemizer charitable deduction. This overstated ordinary taxable income at $113,709 versus $112,571.70."
+us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied 2025 brackets (11,925/48,475/103,350) and a guessed $15,400 standard deduction instead of 2026's $16,100 deduction and 12,400/50,400/105,700 brackets. It also left out the $387 non-itemizer charitable cash deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It placed the $40,884 of qualified dividends and LTCG in the 0% capital gains bracket. Those gains sit on top of $112,572 of ordinary income, far above the roughly $49,450 0% threshold, so all of them are taxed at 15% ($6,132.56). It also missed the $387 charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its own reasoning said tax before refundable credits is positive, but it submitted $0. The correct result is $19,615 of ordinary-rate tax plus $6,133 of preferential-rate tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized deductions such as mortgage interest, real estate taxes and medical expenses, but those total less than the $16,100 standard deduction. Its $16,811 is about $8,900 below the correct total, which is $19,615 of ordinary tax on $112,571.70 plus $6,133 at 15% on $40,884 of preferential income."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It correctly used $169,943 of AGI, the $16,100 standard deduction and 15% preferential rates, but missed the 2026 non-itemizer charitable cash deduction. Subtracting the $387 of cash gifts reduces ordinary taxable income to $112,571.70 and the tax by about $93."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly applied the $387 non-itemizer charitable deduction, but its total is about $137 below $25,747.77. That gap comes from an arithmetic error in the 24%-bracket tax on ordinary taxable income of $112,571.70, which should be $19,615.21."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used only the $16,100 standard deduction and missed the separate non-itemizer charitable cash deduction for 2026. The $387 of cash donations lowers taxable income to $153,455.47 and the tax to $25,747.77."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It took only the $16,100 standard deduction and left out the $387 non-itemizer charitable cash deduction that OBBBA added for 2026. That left ordinary taxable income at $112,959 instead of $112,571.70."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It took $6,889 of employer premiums out of AGI, giving $163,054 instead of $169,942.94. It also claimed a $9,000 overtime deduction, but no overtime premium was given and unlisted amounts are 0. Both errors understated taxable income."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It missed the 2026 non-itemizer charitable cash deduction. Subtracting the $387 of cash gifts on top of the $16,100 standard deduction lowers ordinary taxable income to $112,571.70 and the tax to $25,747.77."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It made up an $8,400 qualified overtime deduction from the hours worked, but no overtime premium was given and unlisted amounts are 0. It also left out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $6,889 of employer-sponsored premiums from wages as if they were salary reduction, giving $163,054 of AGI. Only the $4,245 traditional 401(k) deferral reduces wages, so AGI is $169,942.94 and ordinary taxable income is $112,571.70."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It assumed itemized deductions and 'standard nonrefundable credits' that do not exist for this filer. The itemizable items total less than the $16,100 standard deduction, and no nonrefundable credits apply, so the full $25,747.77 stands."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied rules from a TCJA expiration that did not happen: pre-TCJA brackets, a personal exemption, and miscellaneous deductions above a 2% floor. Under OBBBA, 2026 has the $16,100 standard deduction, the $387 non-itemizer charitable deduction and the 10/12/22/24% brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA had expired, using the 10/15/25% brackets, a $5,300 personal exemption and 2%-floor employee expenses. OBBBA made the TCJA structure permanent, so the correct calculation is the $16,100 standard deduction plus the $387 charitable deduction with 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used rules from a TCJA expiration that did not happen: a $5,450 personal exemption, 2%-floor employee expenses, and the 10/15/25% brackets. Under OBBBA the correct calculation is the $16,100 standard deduction plus the $387 non-itemizer charitable deduction with 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a personal exemption, a 2%-floor miscellaneous deduction and the 10/15/25% brackets. It also deducted the $198 IRA contribution, which phases out for an active plan participant at this MAGI."
+us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized using $5,331 of miscellaneous employee expenses above a 2% floor and an estimated Ohio income tax. OBBBA permanently eliminated miscellaneous itemized deductions, so the remaining itemized items fall short of the $16,100 standard deduction. The correct path is the standard deduction plus the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no usable value and no reasoning for federal_income_tax_before_refundable_credits.
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It claimed an $8,425.50 qualified overtime deduction based on an estimated overtime premium, but no overtime premium was given and unlisted amounts are 0. It also deducted the $198 traditional IRA contribution, which is fully phased out at this MAGI for an active participant."
+us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It claimed a $0 PolicyEngine result that it never computed. A single filer with $153,455.47 of taxable income owes $19,615 of ordinary-rate tax plus $6,133 of 15% tax on preferential income."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $26,973 but submitted $43,879.20, which has no support in its own reasoning. It also never subtracted the $4,245 traditional 401(k) deferral from wages, used a $15,750 standard deduction, and left out the $387 charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $113,617 AGI drops about $56,000 of income from the correct $169,942.94. It made up a QBI deduction even though there is no qualified business income, and it subtracted $194.77 of nonrefundable credits that do not apply."
us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $6,889 employer-sponsored insurance premium from gross wages as a pre-tax cafeteria-plan deduction, giving $93,111 of FICA wages. No Section 125 plan is listed, and PolicyEngine does not reduce the FICA base for this premium. The FICA base is the full $100,000, so the tax is $6,200 + $1,450 = $7,650."
-us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model reduced FICA wages to $93,111 by subtracting the $6,889 employer-sponsored insurance premium as though it were a Section 125 salary reduction. Payroll tax applies to the full $100,000 of gross wages. That gives $6,200 of Social Security tax plus $1,450 of Medicare tax, or $7,650."
-us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model applied 7.65% to $93,111 after subtracting the $6,889 employer-sponsored insurance premium. The premium input is not a pre-tax cafeteria-plan reduction of FICA wages. Applying 7.65% to the full $100,000 gives $7,650."
-us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the $6,889 employer-sponsored insurance premium as a pre-tax health deduction and computed FICA on $93,111. The correct base is the full $100,000 of gross wages, which gives $6,200 of Social Security tax plus $1,450 of Medicare tax, or $7,650."
-us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model shrank the FICA base to $93,111 by subtracting the $6,889 employer-sponsored health premium, a Section 125 exclusion the household facts never establish. With 6.2% and 1.45% applied to the full $100,000 of wages, the total is $7,650."
-us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model excluded the $6,889 employer-sponsored insurance premium from FICA wages as pre-tax health insurance. PolicyEngine taxes the full $100,000 of gross wages at 7.65%, which gives $7,650 rather than $7,122.99."
-us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model's stated method, 6.2% Social Security plus 1.45% Medicare on wages with no Ohio payroll tax, yields exactly $6,200 + $1,450 = $7,650 on $100,000 of wages. Its submitted $8,210 is $560 higher than its own method produces. The extra amount comes from a calculation error, since nothing else in its method applies."
-us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model subtracted the $4,245 traditional 401(k) deferral and computed FICA on $95,755. Elective 401(k) deferrals remain FICA wages under IRC §3121(v)(1)(A). The full $100,000 is taxable, which gives $7,650."
-us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model applied 7.65% to $93,111 after treating the $6,889 employer-sponsored insurance premium as a pre-tax deduction from FICA wages. PolicyEngine's payroll-tax base is the full $100,000 of gross wages, so the correct total is $7,650."
-us,scenario_110,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model correctly kept the traditional 401(k) deferral in FICA wages. It still subtracted the $6,889 employer-plan premium as a pre-tax cafeteria-plan exclusion, giving $93,111. The FICA base is the full $100,000 of gross wages, which yields $7,650."
-us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model excluded the $6,889 employer-sponsored health premium from FICA wages and taxed only $93,111. Gross wages of $100,000 are fully subject to 6.2% Social Security and 1.45% Medicare tax, which gives $7,650."
-us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so no answer could be scored. The correct derivation is 7.65% of the $100,000 in wages, or $7,650."
-us,scenario_110,payroll_tax,minimax-m3,llm_error,other,False,"The model claimed PolicyEngine returned $0 and never applied FICA to the head's $100,000 of wages. Employee Social Security tax of $6,200 and Medicare tax of $1,450 apply to those wages, for a total of $7,650."
-us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The explanation correctly derives $6,200 + $1,450 = $7,650, but the model submitted $9,351.90, which contradicts its own stated total. The error lies in the submitted value, not in the tax rules it described."
-us,scenario_110,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model added $252.55 of Additional Medicare Tax. That 0.9% tax applies only to Medicare wages and self-employment income above $200,000 for a single filer, and investment income does not count toward that threshold. With $100,000 of wages none is owed, so the total is $7,650."
+us,scenario_110,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its own reasoning says employee Social Security and Medicare tax on $100,000 of wages is about $7,650, yet it submitted 0. Nothing in the facts zeroes out FICA on $100,000 of W-2 wages, so the submitted value contradicts its own correct calculation."
+us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It treated the $6,889 employer-sponsored insurance premium as a pre-tax cafeteria-plan reduction and cut FICA wages to $93,111. The facts list no Section 125 salary reduction, so the 7.65% FICA rate applies to the full $100,000 of gross wages, which gives $7,650."
+us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It subtracted the $6,889 employer-sponsored insurance premium from wages to get FICA wages of $93,111. The payroll tax base is the full $100,000 of gross wages: $6,200 Social Security plus $1,450 Medicare, for $7,650."
+us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut FICA wages by the $6,889 employer-sponsored health premium and applied 7.65% to $93,111. The payroll tax base is the full $100,000 of listed gross wages, which gives $7,650."
+us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $6,889 health premium was paid pre-tax and lowered FICA wages to $93,111. Social Security and Medicare tax apply to the full $100,000 of gross wages ($6,200 + $1,450 = $7,650)."
+us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It excluded the $6,889 employer-sponsored premium from FICA wages, giving a base of $93,111. The correct base is the full $100,000 of gross wages, so employee FICA is $7,650."
+us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It subtracted $6,889 of pre-tax health insurance to get FICA wages of $93,111. The 7.65% employee FICA rate applies to the full $100,000 of gross wages, which gives $7,650."
+us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It stated the correct rule (6.2% up to the wage base plus 1.45% Medicare on wages only), but its $8,210 equals 7.65% of about $107,320. That means it inflated the wage base above the $100,000 of listed wages. Applying its own rule to $100,000 gives $7,650."
+us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $4,245 traditional 401(k) deferral and applied FICA to $95,755. Under IRC §3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base is the full $100,000 and the tax is $7,650."
+us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It reduced FICA wages by $6,889 of 'pretax' employer-sponsored insurance premiums to $93,111. The facts give no salary-reduction election, so 7.65% applies to the full $100,000 of gross wages, which gives $7,650."
+us,scenario_110,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the 401(k) deferral in FICA wages but subtracted the $6,889 employer-plan premium, leaving a $93,111 base. The payroll tax base is the full $100,000 of gross wages, so employee FICA is $7,650."
+us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It excluded $6,889 of employer-sponsored health premiums from FICA wages, which left $93,111. The correct base is the full $100,000 of gross wages, so Social Security is $6,200, Medicare is $1,450, and the total is $7,650."
+us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for payroll_tax. The correct output is $7,650 of employee FICA on $100,000 of wages."
+us,scenario_110,payroll_tax,minimax-m3,llm_error,other,False,"It claimed without basis that PolicyEngine returns 0 and never computed anything. $100,000 of W-2 wages carries $6,200 of employee Social Security tax and $1,450 of Medicare tax, for a total of $7,650."
+us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly totals $6,200 Social Security plus $1,450 Medicare as $7,650, but it submitted $9,351.90. That value matches no step in its own calculation."
+us,scenario_110,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It added $252.55 of Additional Medicare Tax, which is 0.9% on about $28,061 above $200,000, so it counted non-wage investment income toward the threshold. The 0.9% Additional Medicare Tax applies only to wages and self-employment income over $200,000, and wages here are $100,000. The correct total is $7,650."
us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own working reached $4,265.55, using the old $360.69 base, a $1,700 exemption and a $169,745 AGI that left the $6,889 pre-tax employer premiums in wages. It then submitted $5,052, which no step of its reasoning supports. The correct 2026 figure is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI instead of $163,421.64, because it did not take the $6,889 pre-tax employer premiums out of wages, so taxable income came to $168,043 instead of $161,521.64. The two errors partly cancel, and the answer ends up $152 low."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $15,000 federal standard deduction, but Ohio has none; the only reduction from Ohio AGI at this income is the $1,900 personal exemption. It then guessed from vague 'progressive brackets' instead of using the 2026 schedule of $332 + 2.75% above $26,050. Its $169,745 AGI also left the $6,889 pre-tax employer premiums in wages."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used an old two-bracket schedule (2.75% up to about $103,500, 3.5% above) to reach about $4,328, then dropped that result for an unexplained $2,900. The 2026 schedule has no 3.5% bracket: tax is $332 + 2.75% × $135,471.64 = $4,057.47. Its $167,895 taxable income was also too high because it left the $6,889 pre-tax employer premiums in wages."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It taxed all income above $26,050 at 3.5% on top of the old $360.69 base, reached $5,325, and then submitted an unrelated $4,517. The 2026 rate above $26,050 is 2.75% plus a $332 base. Its $167,895 taxable income also kept the $6,889 pre-tax employer premiums in AGI and used a $1,850 exemption instead of the $1,900 tier."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It used the old 2.75%/3.5% schedule and took a $2,500 exemption instead of $1,900. It also subtracted the $954 of tax-exempt interest from an AGI that never included it. Its $3,900 equals 2.75% of about $141,800 above $26,050 with no $332 base, taxed on an AGI that still includes the $6,889 pre-tax employer premiums."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,795 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also used a $169,745 AGI, which leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $10,915 of itemized deductions, but Ohio allows none; only the $1,900 personal exemption applies. It used a 3.5% rate above $26,050 when the 2026 rule is $332 + 2.75%. It also took the $20 exemption credit, which is only available when the Ohio income tax base is under $30,000."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It invented a 3.99% bracket, used the old 2.75%/3.5% schedule, and claimed a retirement income credit even though the filer is 51 with no retirement income. It then submitted $3,200, which is below even its own bracket math. The 2026 tax is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It denied the personal exemption, but Ohio allows $1,900 at MAGI above $80,000. It also left out the $332 base and taxed $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the full $169,745 with no personal exemption, claiming a standard deduction and exemption were phased out. Ohio has no standard deduction, and its exemption is still $1,900 at this income. It also used an old multi-bracket schedule (2.765%/3.302%/3.688%) instead of the 2026 schedule of $332 + a single 2.75% rate above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied 2.75% to every dollar of $164,008, ignoring the 0% bracket on the first $26,050 and the $332 base, and it gave no $1,900 personal exemption. The 2026 computation is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its $161,910 of taxable income was close to the correct $161,521.64, but it used the old 2024 schedule (2.75% up to $100,000, 3.5% above) with no base amount. The 2026 schedule taxes all income above $26,050 at 2.75% plus a $332 base, which gives about $4,068 even on the model's own taxable income."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used the $2,400 exemption, which only applies at MAGI of $40,000 or less, instead of $1,900. It kept Ohio AGI at $169,943 instead of $163,421.64. It also applied a made-up three-tier schedule when the 2026 Ohio tax is $332 + 2.75% above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied the 2024 schedule (2.75% up to $100,000, 3.5% above) without base amounts and took a $2,500 exemption instead of $1,900. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums. The 2026 rule is $332 + 2.75% on everything above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted 'standard deductions', but Ohio has none; the only reduction from the $163,421.64 Ohio AGI is the $1,900 exemption. Under the correct $332 + 2.75% schedule, its $2,981 implies about $39,000 less taxable income than the correct $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It started from a $163,054 AGI, close to the $163,421.64 Ohio AGI, but skipped the $1,900 personal exemption. It then used the old 2.75%/3.5% schedule with no base: $2,033.63 + 3.5% × $63,054 ≈ $4,241. The 2026 schedule has no 3.5% bracket; the tax is $332 + 2.75% above $26,050 on $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $169,943, which leaves the $6,889 pre-tax employer premiums in wages, and used an old graduated schedule topping out at 3.5%. For 2026, Ohio replaced that with $332 + a single 2.75% rate above $26,050. Its $3,850 also does not follow from its own stated brackets, which give about $4,400."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47, and $3,280 is $777 short. That implies about $28,300 less taxable income, which fits subtracting deductions Ohio does not allow; Ohio permits only the $1,900 exemption here."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,240.52 is exactly $2,033.63 + 3.5% × ($163,054 − $100,000). It taxed a $163,054 AGI with no $1,900 exemption under the old 2.75%/3.5% schedule. The 2026 tax is $332 + 2.75% above $26,050 on $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47. Its $3,986 equals 2.75% of about $144,950 above $26,050 with no $332 base, which fits taxing an overstated income near $171,000 that keeps the $6,889 pre-tax employer premiums in AGI and skips the $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly said no personal exemption is allowed above $80,000; Ohio allows $1,900 at that income. It also left out the $332 base and taxed $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It said the personal exemption is fully phased out, but Ohio still allows $1,900 at MAGI above $80,000. It kept Ohio AGI at $169,943 instead of $163,421.64 and applied the 2024 2.75%/3.5% schedule. For 2026, all income above $26,050 is taxed at 2.75% plus a $332 base."
-us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $169,943 with no personal exemption under the old 2.75%/3.5% schedule. The 2026 rule is $332 + 2.75% above $26,050, applied to $161,521.64 ($163,421.64 Ohio AGI less the $1,900 exemption)."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It acknowledged that Ohio tax is positive at this income but refused to compute it and entered $0. Applying the $1,900 exemption and the 2026 schedule of $332 + 2.75% above $26,050 to $163,421.64 of Ohio AGI gives $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted itemized deductions (mortgage interest and charitable gifts), which Ohio does not allow, and still reported $5,625. Even a flat 2.75% on the whole $169,943 comes to only $4,673, so its figure implies a rate well above Ohio's single 2026 rate of 2.75%. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,993 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,418 matches the old 2024 schedule ($2,033.63 on the first $100,000 plus 3.5% above) applied to about $168,100 of taxable income. That income comes from a $169,943 AGI that includes the $6,889 pre-tax employer premiums. The 2026 schedule is $332 + 2.75% above $26,050 on $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied only 2.75% above the zero bracket and left out the $332 base in the 2026 Ohio schedule. Its $3,905 corresponds to $168,043 of taxable income, built from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base and used a $1,850 exemption instead of the $1,900 tier for MAGI above $80,000. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $1,900 exemption and the $332 + 2.75% schedule correctly. However, it built Ohio AGI as $163,054 plus the full $954 of tax-exempt interest, giving $164,008, which is $586.36 above the $163,421.64 Ohio AGI. That put taxable income at $162,108 instead of $161,521.64 and the tax $16.13 too high."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $169,943 with no personal exemption under the old 2.75%/3.5% schedule. The 2026 rule is $332 + a single 2.75% rate above $26,050, applied to $161,521.64 of taxable income after the $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It skipped the $1,900 personal exemption and left out the $332 base, taxing $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $1,900 exemption and the $332 + 2.75% schedule correctly, but treated Ohio AGI as equal to its $163,054 federal AGI. That left out the net +$367.64 of Ohio adjustments that bring Ohio AGI to $163,421.64. Taxable income came to $161,154 instead of $161,521.64, and the tax is $10.11 too low."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation and claimed 'modest credits', but no Ohio nonrefundable credit applies here; the $20 exemption credit requires an Ohio income tax base under $30,000. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47, so $3,200 is $857 short."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly used the 2024 schedule (2.75% up to $100,000, 3.5% above) with no base amounts on $168,043. The 2026 schedule has no 3.5% bracket; it is $332 + 2.75% above $26,050. Its $169,943 AGI also leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,993 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied the old 2.75%/3.5% schedule to $168,997: $2,033.63 + 3.5% × $68,997 ≈ $4,449. The 2026 rule is $332 + a single 2.75% rate above $26,050. Its income base also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base and used the $2,400 exemption, which only applies at MAGI of $40,000 or less, instead of $1,900. It also started from a $169,745 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2024 schedule: about $2,394 of tax on the first $100,000 plus 3.5% on the excess. For 2026, Ohio replaced that with $332 + a single 2.75% rate above $26,050. Its taxable income of about $167,500 also came from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base in the 2026 schedule and subtracted the $20 exemption credit, which is only available when the Ohio income tax base is under $30,000. It also started from a $169,745 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It made up a claim that PolicyEngine returns $0 and did no computation. Ohio AGI of $163,421.64, less the $1,900 exemption, taxed at $332 + 2.75% above $26,050, gives $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base in the 2026 schedule and took a $1,700 exemption instead of the $1,900 tier for MAGI above $80,000. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $954 of tax-exempt interest in AGI and did not subtract the $4,245 401(k) deferral from gross wages. It then used made-up 2.765%/3.225% brackets, an invented $250 exemption credit and an unexplained further 'graduated credit', ending at $3,148.93. The 2026 tax is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It cut Ohio taxable income to about $96,517 by subtracting 'the standard deduction and exemptions', but Ohio has no standard deduction and allows only a $1,900 exemption. The correct taxable income is $161,521.64, and $332 + 2.75% above $26,050 gives $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the 2024 base of $360.69 instead of the 2026 base of $332, a ~$1,700 exemption instead of $1,900, and an AGI that deducted the phased-out $198 IRA, which gave $4,265.55. It then submitted $5,052, a number its own derivation does not support."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its taxable income of $168,043 is correct, but it applied 2.75% only to the amount above $26,050 and left out the $332 base in the 2026 Ohio schedule. That gives $3,905 instead of $332 + $3,904.81 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a federal-style $15,000 standard deduction, which Ohio does not have, instead of the $1,900 personal exemption. It also deducted the phased-out IRA contribution and guessed at progressive brackets instead of applying $332 + 2.75% above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It submitted $0 without doing any calculation, even though $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used invented indexed brackets with a 3.5% rate above $103,500, which no longer exists in 2026, and reached ~$4,328. It then dropped to an arbitrary $2,900 with no derivation instead of applying $332 + 2.75% over $26,050 to $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 3.5% to all income above $26,050 on top of the 2024 base of $360.69, which gave $5,325. It then submitted an unexplained $4,517. The 2026 schedule is a flat 2.75% plus a $332 base, applied to $168,042.94 after the $1,900 exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted the $954 tax-exempt interest even though it was never in AGI, and used a ~$2,500 exemption instead of $1,900. Its $3,900 matches a flat 2.75% above $26,050 with the $332 base left out."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule, taxing only 2.75% of the excess over $26,050. It also deducted the $198 traditional IRA contribution, which is fully phased out given workplace-plan coverage and AGI near $170k, so its taxable income came out $198 too low."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted $10,915 of itemized deductions, which Ohio does not allow, instead of the $1,900 personal exemption. It applied 3.5% instead of the 2026 rate of 2.75% plus the $332 base, and took the $20 exemption credit, which is unavailable at this income."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used invented graduated brackets (3.5% and ~3.99%) instead of the 2026 schedule of $332 + 2.75% above $26,050. It also subtracted nonrefundable credits (retirement income and others) that this household does not qualify for, ending at $3,200."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It wrongly skipped the personal exemption, which is $1,900 for income above $80k rather than zero. It also left out the $332 base and taxed only 2.75% of AGI above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used old graduated rates (2.765%/3.302%/3.688%) that have been repealed, deducted the phased-out IRA, and wrongly treated the $1,900 exemption as phased out. The 2026 schedule is $332 + 2.75% above $26,050 on $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied 2.75% to the entire taxable income with no $26,050 zero bracket and no $332 base, and left out the $1,900 exemption. Its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums, and it added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $162,856 subtracts both the $6,889 employer insurance premiums and the phased-out $198 IRA from the correct $169,942.94. It then applied the repealed 2.75%/3.5% two-bracket schedule without the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $2,400 exemption instead of the $1,900 that applies above $80k, and invented a three-component graduated calculation instead of $332 + 2.75% × ($168,042.94 − $26,050)."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the 3.5% rate above $100,000, which was eliminated for 2026, without the $332 base. It also used a $2,500 exemption instead of $1,900."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It says it subtracted 'standard deductions', but Ohio has none; Ohio allows only the $1,900 personal exemption. Its $2,981 is $1,256 short, which matches subtracting about $34k of nonexistent deductions and leaving out the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $163,054, which wrongly subtracts the $6,889 employer insurance premiums from wages; the correct AGI is $169,942.94. It applied progressive brackets instead of the 2026 schedule of $332 + 2.75%, and its $4,241 lands near the reference only because these errors offset."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a progressive schedule with a 3.5% top rate, which has been repealed, instead of the 2026 schedule of $332 + 2.75% above $26,050. That produced $3,850 instead of $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. Its $3,280 is $957 short of $332 + 2.75% × ($168,042.94 − $26,050), which matches leaving out the $332 base and subtracting deductions beyond the $1,900 exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,240.52 is $3.71 above the correct $332 + 2.75% × $141,992.94, which means it taxed about $135 more than the correct $168,042.94 of taxable income."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no derivation. Its $3,986 matches 2.75% above $26,050 on about $171k with no $1,900 exemption and no $332 base, instead of $332 + 2.75% × ($168,042.94 − $26,050)."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It wrongly claimed there is no personal exemption above $80,000; the exemption is $1,900 at that level. It also left out the $332 base, taxing only 2.75% of AGI above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the repealed 3.5% rate above $100,000 and left out the $332 base. It also wrongly treated the personal exemption as fully phased out, when $1,900 applies above $80k."
+us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2024-style schedule with 3.5% above $100,000, which is repealed for 2026, and left out the $332 base. It also skipped the $1,900 personal exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It admitted the tax is positive but submitted $0 anyway, instead of computing $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted itemized deductions (mortgage interest, charity), which Ohio does not allow; Ohio allows only the $1,900 personal exemption. Its $5,625 overshoots by $1,388, which matches graduated rates above the 2026 flat 2.75% plus $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly found $168,043 of taxable income, but taxed only 2.75% of the excess over $26,050 and left out the $332 base in the 2026 schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $4,418 matches the repealed 2.75%/3.5% two-bracket schedule without the $332 base, applied to roughly $168k of taxable income. The 2026 rate is a flat 2.75% plus $332 above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied 2.75% only to income above the zero bracket and left out the $332 base in the 2026 Ohio schedule, which puts it $332 low."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule and used a $1,850 exemption instead of $1,900."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied the $332 + 2.75% schedule correctly, but its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums. It also added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include, leaving taxable income at $162,108 instead of $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It used the repealed 3.5% bracket above $100,000 without the $332 base. It also skipped the $1,900 personal exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption and left out the $332 base, taxing only 2.75% of AGI above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its rate schedule ($332 + 2.75% above $26,050) and $1,900 exemption are correct. But its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums from wages, which understates taxable income by $6,889."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no real derivation and subtracted 'modest credits' this household does not qualify for. Its $3,200 is $1,037 short of $332 + 2.75% × ($168,042.94 − $26,050)."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its $168,043 of taxable income is correct, but it applied the 2024-style 3.5% bracket above $100,000, which is repealed for 2026, and left out the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly found $168,043 of taxable income, but computed only 2.75% × $141,993 and left out the $332 base in the 2026 schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include. It also applied the repealed 3.5% bracket above $100,000 without the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It left out the $332 base. It also used a $2,400 exemption instead of the $1,900 that applies above $80k, and deducted the phased-out $198 IRA from AGI."
+us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the 2024 schedule ($2,394 at $100,000 plus 3.5% above), but for 2026 the 3.5% bracket is gone and all income above $26,050 is taxed at 2.75% plus a $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for state_income_tax_before_refundable_credits.
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It left out the $332 base and subtracted the $20 personal exemption credit, which is limited to low-income filers. It also deducted the phased-out $198 IRA from AGI."
+us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It falsely claimed PolicyEngine returned $0. The $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule and used a $1,700 exemption instead of the $1,900 that applies above $80k."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It added $954 of tax-exempt interest into AGI and used invented 2.765%/3.225% brackets. It then subtracted an invented $250 exemption credit and further invented 'rate reductions', instead of $332 + 2.75% above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It cut taxable income to $96,517 by subtracting about $71,500 of standard deductions that Ohio does not have. The correct base is $169,942.94 AGI minus only the $1,900 exemption, which is $168,042.94."
us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate."
us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount."
-us,scenario_111,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated aged, blind, or disabled status plus low bank assets as sufficient for Washington Medicaid and asserted without calculation that disregards reduced $43,640 below the senior income limit. It also invoked the $65 earned-income disregard and work incentives even though the head has no earnings; the head instead qualifies through no Medicaid category."
-us,scenario_111,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model conflated the MAGI expansion pathway with aged/blind/disabled pathways and assumed that Social Security exclusion and low assets created eligibility. The traced MAGI is 2.02 times FPL, above the relevant MAGI limits, and aged/blind/disabled status produces no separate qualifying category here."
-us,scenario_111,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed that blind and disabled status automatically opened an SSI-related Medicaid pathway despite zero SSI receipt, then treated listed medical expenses as a spend-down that established eligibility. PolicyEngine assigns no aged/blind/disabled or other Medicaid category to the head, so the resource-limit and vehicle-exclusion discussion never establishes eligibility."
+us,scenario_111,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model wrongly said seniors are tested under MAGI rules. It also applied the $65 earned-income disregard even though the head has no earned income. Only the $20 general disregard applies, which leaves about $3,617 a month of countable income. That is far above Washington's aged/blind/disabled income limit, yet the model claimed the income fell below it without doing the comparison."
+us,scenario_111,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said that being aged and disabled makes the head eligible, but it never ran the income test. The head has $43,640 a year (about $3,637 a month) in Social Security and IRA income and receives no SSI. That is far above the aged/disabled income limit, so no Medicaid category applies."
+us,scenario_111,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used Washington's MAGI expansion pathway, which only covers adults aged 19–64 with income up to 138% FPL. The head is 72 and has MAGI of about 202% FPL. The model also counted low assets and aged/disabled status as enough for the aged pathway without checking the head's roughly $3,637 monthly income against its income limit."
+us,scenario_111,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model relied on a spend-down, using SSI-type disregards and medical expenses to bring $43,640 of income under the aged/blind/disabled limit. The only disregard is $20 a month, and the head's medical and OTC expenses total just $1,500 a year. That leaves countable income of about $3,617 a month, far above the categorical aged/blind/disabled limit. Passing the asset test alone does not establish eligibility."
us,scenario_111,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages, then contradicted that reasoning by imposing $1,046.85 of mandatory payroll tax on retirement income. Social Security retirement benefits and taxable IRA distributions are not employee payroll-tax bases, so the absence of employment compensation yields payroll_tax of $0."
us,scenario_111,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated income below Washington's 200% gross-income screen as categorical eligibility and then awarded the $24 monthly minimum. The household fails the applicable income eligibility determination, so it is ineligible rather than an eligible household whose calculated allotment is raised to the minimum."
-us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness or disability as conferring Medicaid eligibility regardless of income. Those statuses only open disability-related pathways, and the spouse does not satisfy any pathway’s remaining eligibility requirements."
-us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned the spouse zero countable income under spousal-income rules and used that assertion to approve an ABD/SSI-related pathway. The applicable eligibility calculations leave the spouse in no Medicaid category, while household MAGI is 2.02 times FPL and fails the expansion pathway."
-us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model jumped from the spouse’s blindness and disability to qualification under an ABD pathway without applying that pathway’s income and other eligibility tests. Disability creates a potential category, not automatic coverage, and the spouse qualifies through none of the available categories."
-us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model acknowledged that household income exceeds the ACA expansion limit but then treated blindness or disability as sufficient for SSI-related or medically needy eligibility. Neither status waives the separate requirements of those pathways, and the spouse receives no SSI and qualifies through no Medicaid category."
-us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model asserted that the head’s Social Security and IRA income was low enough for the blind/disabled income limit without calculating or applying that limit. Excluding one vehicle and satisfying a resource ceiling does not cure failure of the applicable income and categorical tests, so no ABD pathway is established."
-us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model characterized countable income as low after unspecified exclusions and then approved ABD Medicaid without computing the controlling income test. The spouse’s disability and low stated resources do not independently establish eligibility, and no ABD or other non-MAGI category is satisfied."
-us,scenario_111,spouse_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model understated Medicaid MAGI by counting only the $18,240 IRA distribution and excluding all Social Security on the ground that none was federally taxable. The eligibility computation produces MAGI equal to 2.02 times FPL, which is above the 138% FPL adult expansion limit."
-us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the spouse’s lack of personally assigned income as zero income for ABD Medicaid and also understated household MAGI as $18,240. The applicable calculations yield MAGI of 2.02 times FPL and do not place the spouse in an ABD or any other Medicaid category; excluding the vehicle and noting $840 of bank assets does not establish eligibility."
+us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model claimed that disability-based Medicaid is not income-limited and that the spouse has no income to count. Blind/disabled Medicaid is tied to SSI receipt or an SSI-related income test, and the head's $43,640 in Social Security and IRA income is deemed to the spouse. That leaves SSI at $0 and no disability category, and MAGI of 2.02x FPL also fails the 138% expansion limit."
+us,scenario_111,spouse_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said household income was low relative to the disability Medicaid limit but never compared any numbers. The couple's combined countable income is $43,640, which is far above the SSI couple standard and 2.02x FPL. The spouse gets no SSI and fails both the disability pathway and the 138% FPL adult-expansion test."
+us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly found that $43,640 exceeds the 138% FPL expansion limit. It then claimed the spouse qualifies through disability because they have no income of their own, which ignores spousal deeming in the SSI-related pathway. Once the head's income is counted, the spouse's SSI is $0 and no disability-based Medicaid category applies."
+us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model said the spouse qualifies for disability-related Medicaid because they are blind and disabled but never ran the income test. The SSI-related test counts the couple's combined $43,640 Social Security and IRA income, which exceeds the limit, so SSI is $0. MAGI of 2.02x FPL also exceeds the 138% expansion threshold, which leaves the spouse with no eligibility category."
+us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly found that household income exceeds the 138% FPL expansion limit. It then asserted that blindness/disability makes the spouse eligible regardless of income. PolicyEngine's disability pathway requires SSI receipt or passing an SSI-related income test that counts spousal income. The head's $43,640 brings the spouse's SSI to $0, so no category applies."
+us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model checked the resource test but then asserted, without any arithmetic, that the head's Social Security and IRA income was low enough for the blind/disabled income limit. The combined $43,640 of countable couple income exceeds the SSI-related couple standard, so the spouse gets $0 SSI. It also equals 2.02x FPL, which fails the 138% expansion limit."
+us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model claimed Social Security is not fully counted and that countable income would be low after exclusions. The SSI-related test counts gross Social Security, and Medicaid MAGI adds back non-taxable benefits. Countable couple income is the full $43,640, or 2.02x FPL, which fails both the blind/disabled test and the 133–138% expansion limit."
+us,scenario_111,spouse_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model left the non-taxable Social Security out of MAGI and used only $18,240, or 84% FPL. Medicaid MAGI adds back the non-taxable portion of Social Security benefits (42 CFR 435.603(e)). The correct MAGI is therefore $43,640, or 2.02x FPL, which exceeds the 138% adult-expansion limit."
+us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used $18,240 as MAGI, leaving out the head's $25,400 Social Security, which Medicaid MAGI adds back. The true MAGI is $43,640, or 2.02x FPL, above the 138% limit. Its blind/disabled fallback treated the spouse as having $0 income, which ignores deeming of the head's income. That deeming brings SSI to $0, so the disability category also fails."
us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model claimed that disabled people qualify for Medicare regardless of age. Under-65 disability eligibility actually requires 24 months of Social Security disability benefit entitlement. The 49-year-old spouse has no SSDI income or SSDI months listed, so the correct answer is not eligible."
us,scenario_111,spouse_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model applied the Medicare disability pathway using only the disabled flag. That pathway requires 24 months of SSDI entitlement, and the spouse has zero Social Security disability benefits, so the pathway does not apply."
us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model stated that Medicare covers disabled people under 65 and stopped there. It never applied the requirement of 24 months of Social Security disability benefit receipt, which the 49-year-old spouse (with no SSDI listed) does not meet."
@@ -8755,27 +8973,28 @@ us,scenario_111,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,Fa
us,scenario_111,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model correctly noted that the spouse is below the age-65 threshold. It then claimed PolicyEngine treats any disabled person as Medicare-eligible regardless of age. The under-65 pathway actually requires 24 months of Social Security disability benefit receipt, which the spouse (with no SSDI) lacks."
us,scenario_112,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model explicitly built AGI as ""wages $5,923 + farm rent $1,920 + financial assistance $13,000 - $3,000"" = $17,843, treating the $13,000 financial-assistance receipt as taxable ordinary income; financial assistance is a non-taxable cash transfer excluded from gross income and AGI, so true AGI is $4,843. Removing that $13,000 leaves AGI below the ~$16,100 single standard deduction, so taxable income is $0 and the 10%-bracket computation that produced $174 never arises."
us,scenario_112,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model stated its ordinary income base as ""wages, farm-rent income, and financial assistance,"" including the $13,000 financial assistance in taxable income even though that receipt is a non-taxable transfer excluded from gross income and AGI. With only $5,923 of wages plus $1,920 of farm rent less the $3,000 allowable capital loss, AGI is $4,843 — under the 2026 single standard deduction — so taxable income is $0 and the $1,743 it ran through the 10% bracket to reach $174.30 is entirely the misclassified assistance."
-us,scenario_112,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model executed the correct derivation in full - childless EITC phase-in of 7.65% on $5,923 of earned income, AGI of $4,843 below the phase-out start, investment income below the disqualification limit - and stated the answer as approximately $453, then submitted 191. The submitted number corresponds to no step in its own arithmetic; it discarded its correct $453.10 result at the point of transcription."
-us,scenario_112,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model reasoned that the capital loss and standard deduction drive taxable income negative, so with no tax liability no refundable credits are available. That inverts the definition of a refundable credit: the EITC under IRC section 32 is paid in full irrespective of liability, and its phase-in base is earned income ($5,923 x 7.65% = $453.10), not taxable income. It also asserted income was below the EITC threshold when $5,923 is squarely on the phase-in ramp, which has no floor."
-us,scenario_112,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model verified every gating condition correctly - age 31 within the 25-64 childless band, capital loss not disqualifying, AGI $4,843 below the phase-out start - and computed $5,923 x 0.0765 = $453, then overrode it with an explicit 'conservative estimate' of $50. The EITC phase-in is a fixed statutory percentage of earned income with no discount for small earnings, so the correct value is its own $453.10 figure."
-us,scenario_112,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the -$9,944 long-term capital loss and 'negative AGI overall' as collapsing the childless EITC to zero. The EITC phase-in applies to earned income of $5,923, capital losses never reduce earned income, and the capital loss offsets ordinary income only up to the $3,000 cap, leaving AGI at a positive $4,843 - below the phase-out start, so the full 7.65% phase-in credit of $453.10 stands."
-us,scenario_112,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly stated that earnings never reach the plateau and that the credit is 7.65% of $5,923 = about $453, then submitted 492 on the contradictory ground that the credit sits 'near the maximum plateau region.' The plateau begins only at the earned income amount of roughly $8,700, where the maximum childless credit of about $664 applies; at $5,923 the credit is strictly the phase-in product, $453.10."
-us,scenario_112,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the $9,944 capital loss as pushing AGI below zero and 'offsetting' the credit, and additionally used $7,843 (wages plus farm rent) as the earned income base. Farm rent is not earned income for the EITC, the capital loss is capped at $3,000 against ordinary income leaving AGI at $4,843, and neither affects the phase-in base of $5,923, which yields exactly 7.65% x $5,923 = $453.10 rather than the $0 submitted."
-us,scenario_112,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,389 employer-sponsored insurance premiums from $5,923 of wages to conclude earned income is $0 for EITC purposes. The section 125 exclusion cannot reduce reported wages below zero and does not net against the gross wage figure given; $5,923 is the earned income base, producing a childless EITC of $453.10."
-us,scenario_112,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model built the calculation correctly - childless EITC, earned income $5,923, AGI $4,843 below the phase-out threshold - but applied a phase-in rate of 7.67% instead of the statutory 7.65% credit percentage for taxpayers with no qualifying children under IRC section 32(b)(1). That rate error alone accounts for the $1.19 overstatement ($454.29 versus $453.10)."
-us,scenario_112,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted the household fails an 'earned income threshold' for the EITC. The childless EITC has no minimum earnings threshold: it phases in from the first dollar at 7.65%, so $5,923 of wages for a 31-year-old non-dependent produces $453.10, not zero."
-us,scenario_112,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model claimed the pre-tax employer-sponsored insurance premiums of $8,389 'fully offset' the $5,923 of gross wages, leaving zero earned income for the EITC. Premium exclusions do not net against the stated gross wage figure and cannot drive wages negative; earned income remains $5,923 and the childless EITC is 7.65% of that, $453.10."
-us,scenario_112,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model deducted the $8,389 in pre-tax health insurance premiums from wages to reach $0 earned income and declared the head ineligible for the EITC. The $5,923 gross wage figure is the EITC earned income base regardless of premium amounts, so the childless EITC phase-in yields $5,923 x 0.0765 = $453.10."
-us,scenario_112,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted zero federal refundable credits with no computation. The head is a 31-year-old non-dependent single filer with $5,923 in wages, AGI of $4,843 and investment income of $1,920, satisfying every childless EITC condition, so the correct value is the 7.65% phase-in amount of $453.10."
-us,scenario_112,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model concluded there is 'zero taxable earned income after pre-tax health deductions' and paired that with the absence of qualifying children to return $0. Health premium exclusions do not reduce the $5,923 wage base used for the EITC, and the childless EITC exists precisely for filers with no children, yielding $453.10 here."
-us,scenario_112,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model correctly identified earned income of $5,923 and AGI of $4,843, then applied the 7.65% phase-in rate to AGI on the stated rule that the EITC uses the lesser of earned income or AGI. The lesser-of rule governs only the phase-out comparison; the phase-in credit is the rate times earned income, so the credit is 0.0765 x $5,923 = $453.10, not 0.0765 x $4,843 = $370.70."
-us,scenario_112,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model computed the correct $453 phase-in amount, then zeroed it by using a modified AGI of $17,843 that includes the $13,000 of financial assistance. Financial assistance is not taxable income and enters neither AGI nor EITC modified AGI; the actual comparison figure is $5,923 of earned income against AGI of $4,843, both below the roughly $10,600 phase-out start, so no reduction applies and the credit is $453.10."
-us,scenario_112,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model concluded that with no qualifying children no refundable federal credit is 'indicated by the provided facts.' The childless EITC applies to workers aged 25-64 with earned income; the head is 31 with $5,923 in wages and AGI below the phase-out start, giving $453.10."
-us,scenario_112,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model stated no refundable credit is 'specified or inferable' because there are no children and no premium tax credit. Wages of $5,923 for a 31-year-old single non-dependent trigger the childless EITC automatically at a 7.65% phase-in rate, which is $453.10."
-us,scenario_112,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted the earned income was insufficient for the EITC in the absence of qualifying children. The childless EITC phases in from the first dollar of earnings with no minimum, so $5,923 in wages produces $453.10; low earnings reduce the credit proportionally rather than eliminating it."
-us,scenario_112,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model characterized the childless EITC at this income level as 'minimal or zero' and rounded the total to $0 without computing it. At $5,923 of wages the phase-in is exact and mandatory: 7.65% x $5,923 = $453.10, a figure the model never calculated."
-us,scenario_112,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model computed the correct $453 phase-in amount, then reduced it to $390 on the theory that the $9,944 capital loss 'counts as negative investment income' and cuts the credit. The investment income test is a pass/fail disqualification against the roughly $12,200 limit, never a reduction of the credit amount, and losses do not scale it down; the credit stands at $453.10."
-us,scenario_112,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted flatly that no refundable federal credits apply, with no analysis. The facts satisfy every childless EITC requirement - age 31, non-dependent single filer, $5,923 in earned income, AGI of $4,843 below the phase-out start, and investment income of $1,920 below the disqualification limit - so the credit is $453.10."
+us,scenario_112,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning correctly arrived at a childless EITC of 7.65% × $5,923 ≈ $453 with no phase-out reduction, but it then submitted 191. Nothing in its own derivation supports that number."
+us,scenario_112,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It assumed EITC needs tax liability or children and that the income was 'below EITC thresholds.' The childless EITC is refundable and phases in at 7.65% from the first dollar of earnings, so $5,923 of wages earns about $453. It also wrongly treated the employer-insurance premiums and the uncapped capital loss as making income negative."
+us,scenario_112,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It said no EITC applies because the head has no qualifying children and very low earnings. A 31-year-old without children qualifies for the childless EITC, which phases in at 7.65% of earned income with no minimum, giving about $453 on $5,923 of wages."
+us,scenario_112,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly found childless EITC eligibility (age 31, AGI $4,843 below the phase-out, no investment-income problem) and computed $5,923 × 7.65% ≈ $453. It then swapped in an arbitrary 'conservative estimate' of $50 that no rule supports."
+us,scenario_112,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It concluded that the capital loss produces negative AGI that drives the EITC to zero. The deductible capital loss is capped at $3,000, so AGI is a positive $4,843 and below the phase-out start. The credit is the 7.65% phase-in on $5,923 of earned income, about $453."
+us,scenario_112,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It first got 7.65% × $5,923 ≈ $453 and then raised the answer to $492, saying the earnings were 'near the maximum plateau.' $5,923 is well below the earned-income amount where the childless credit reaches its maximum, so the credit is simply 7.65% × $5,923 ≈ $453."
+us,scenario_112,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It counted farm rent as earned income and assumed the full $9,944 capital loss pushes AGI below zero, which it said makes the credit negligible. Farm rent is unearned income, and the capital-loss deduction is capped at $3,000, so AGI is $4,843. The credit is the 7.65% phase-in on $5,923 of wages, about $453."
+us,scenario_112,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer-insurance premium from the $5,923 of wages and got zero earned income. The premiums do not reduce the reported wage income used as EITC earned income. The childless EITC is 7.65% × $5,923 ≈ $453."
+us,scenario_112,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 7.67% phase-in rate. The statutory childless EITC phase-in rate is 7.65% and is not indexed for inflation, so the credit is 7.65% × $5,923 ≈ $453.10, not $454.29."
+us,scenario_112,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It claimed the household falls short of an EITC earned-income threshold. The childless EITC has no minimum earnings requirement and phases in at 7.65% from the first dollar, giving about $453 on $5,923 of wages."
+us,scenario_112,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It treated the $8,389 employer-insurance premium as a pre-tax deduction that wipes out the $5,923 of wages. The wage input counts in full as EITC earned income, so the childless EITC is 7.65% × $5,923 ≈ $453."
+us,scenario_112,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reduced earned income to $0 by subtracting the employer-insurance premiums from gross wages. Earned income for the EITC is the full $5,923 of wages, and the 7.65% childless phase-in gives about $453."
+us,scenario_112,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Its $0 answer leaves out the childless EITC entirely. The single 31-year-old head with $5,923 of wages and AGI of $4,843 qualifies for a 7.65% phase-in credit of about $453."
+us,scenario_112,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It treated the employer-insurance premiums as a pre-tax deduction that leaves zero earned income. The full $5,923 of wages is EITC earned income, giving a childless EITC of 7.65% × $5,923 ≈ $453."
+us,scenario_112,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It applied the 7.65% phase-in rate to the smaller of earned income and AGI ($4,843). The phase-in credit is calculated on earned income alone ($5,923 → about $453). AGI only matters for the phase-out, where the greater of AGI and earned income is compared with the threshold."
+us,scenario_112,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It built a 'modified AGI' of $17,843 by adding the $13,000 of nontaxable financial assistance, which pushed the credit into full phase-out. Financial assistance is not in AGI, so AGI is $4,843, below the phase-out start, and the full 7.65% × $5,923 ≈ $453 credit applies."
+us,scenario_112,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated having no qualifying children as ruling out every refundable credit. A 31-year-old worker without children qualifies for the childless EITC, which is 7.65% × $5,923 ≈ $453 here."
+us,scenario_112,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It concluded no refundable credit applies because there are no children. It missed the childless EITC, which a 31-year-old wage earner qualifies for: 7.65% × $5,923 ≈ $453."
+us,scenario_112,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It said earned income is not enough for the EITC. The childless EITC has no earnings floor and phases in at 7.65% from the first dollar, so $5,923 of wages gives about $453."
+us,scenario_112,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It dismissed the childless EITC as 'minimal or zero' at this income level. The credit phases in at 7.65% of earned income and AGI ($4,843) is below the phase-out start, so the full $453 phase-in amount is refundable."
+us,scenario_112,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It computed 7.65% × $5,923 = $453 and then cut it to $390 for a supposed capital-loss effect on investment income. The EITC investment-income test is all-or-nothing: it removes the credit only when investment income exceeds the limit. The farm rent here is far below that limit, so no reduction applies and the credit stays at about $453."
+us,scenario_112,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It concluded no refundable credits apply and missed the childless EITC. The single 31-year-old head with $5,923 of wages and $4,843 of AGI qualifies for a 7.65% phase-in credit of about $453."
us,scenario_112,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated low MAGI as a standalone Texas Medicaid pathway for a childless adult. A 31-year-old nondependent must first belong to an active eligibility category, and this head belongs to none."
us,scenario_112,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility directly from low earned income and minimal assets without identifying a Texas eligibility category. Assets do not establish a MAGI pathway, and this childless, nondependent adult qualifies through none of the available categories."
us,scenario_112,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly assumed that low income qualifies a single Texas adult for Medicaid. Texas has no general MAGI pathway for this 31-year-old childless, nondependent adult, whose Medicaid category is NONE."
@@ -8793,98 +9012,100 @@ us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,
us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings."
us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies."
us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271."
-us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"It counted the $160/month of farm rent, which gave $1,737/month gross, and failed a 130% gross test that Texas replaces with its 165% BBCE limit; its fallback net figure of about $1,429 also includes the farm rent. The reference's countable income is wages plus financial assistance ($1,576.92), so net income is $1,269.19, below the $1,304 limit, and the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"It applied Texas BBCE correctly but counted the $160/month of farm rent, which raised net income to about $1,429, above the 100% poverty-guideline limit. The reference's income excludes farm rent, so net income is $1,269.19 and the test is passed. A one-person household that passes still gets the $23.84 minimum allotment even when 30% of net income is more than the $298 maximum."
-us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $1,737/month gross and failed a gross income limit, ignoring Texas's 165% BBCE limit. The reference's gross income is $1,576.92 without farm rent (121% of the poverty guideline); net income of $1,269 passes the net test, and the one-person minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"It added the $1,920 of farm rent to reach $1,737/month and failed a 130% gross test of about $1,650, which Texas BBCE replaces with 165%. The reference's countable income leaves out farm rent, net income of $1,269 is below the $1,304 limit, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"It treated the $13,000 of financial assistance as a non-countable gift, but it is countable unearned income ($1,083.33/month), which puts net income at $1,269.19. At that level, 30% of net income ($380.76) is more than the $298 maximum, so only the one-person minimum allotment of $23.84/month is paid, not about $192/month."
-us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"It left the $13,000 of financial assistance out of countable income and got a near-maximum benefit of $275/month. Once the $1,083.33/month of assistance is counted, net income is $1,269.19, 30% of it is more than the $298 maximum, and the household receives only the $23.84/month minimum allotment."
-us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"It assumed the financial assistance is not counted and got a net income of $346 and a benefit of $194/month. SNAP counts the $1,083.33/month of financial assistance as unearned income, which makes net income $1,269.19; the $380.76 expected contribution is more than the $298 maximum, leaving only the $23.84/month minimum allotment."
-us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $20,843/year and applied a 130% gross limit of about $19,584, ignoring Texas's 165% BBCE limit. The reference's countable income is $18,923/year without farm rent; it passes the gross and net tests (net income is $1,269/month), and the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and applied a 130% gross limit (about $1,632/month) instead of Texas's 165% BBCE limit. The reference's countable income, without farm rent, is $1,576.92 gross and $1,269.19 net; this passes the 100% net test and yields the $23.84/month minimum allotment."
-us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"It treated the $13,000 of financial assistance as not countable and got $194/month. Counting the $1,083.33/month of assistance as unearned income puts net income at $1,269.19; 30% of that is more than the $298 maximum, so only the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"It included the $1,920 of farm rent to reach $20,843 and tested that against the 130% limit, not Texas's 165% BBCE limit. The reference's countable income is $18,923/year without farm rent, net income of $1,269/month passes the net test, and the minimum allotment of $23.84/month applies."
-us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Its $1,737/month gross figure includes farm rent, and it applied a 130% gross test in place of Texas's 165% BBCE limit. The reference counts $1,576.92/month without farm rent, which passes the net test at $1,269.19, and it pays the one-person minimum allotment of $23.84/month."
-us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and concluded that because 30% of net income is more than the maximum allotment the benefit is $0, never applying the minimum allotment for one-person households. Without farm rent, net income is $1,269.19, below the $1,304 limit, so the household receives the $23.84/month minimum."
-us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"It computed an annual net income of $6,454 and compared the annual 30% contribution ($1,936) with a monthly maximum allotment, mixing annual and monthly figures. The correct monthly net income is $1,269.19; its $380.76 contribution is more than the $298 maximum, and the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Its net income of $1,440.20 includes the $160/month of farm rent, which puts it over the net limit. The reference's countable income leaves out farm rent, so net income is $1,269.19; that passes the 100% net test, and the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Its $0 answer matches counting every listed income item, including farm rent, which makes the household fail the net test. The reference's countable income is wages plus financial assistance ($1,576.92/month); net income of $1,269.19 passes, and the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"It reasoned that because 30% of net income is more than the maximum, the benefit is $0; this skips the minimum allotment and counts the farm rent. Without farm rent, net income is $1,269.19, below the $1,304 limit, so the one-person household receives the $23.84/month minimum allotment."
-us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"It computed net income of about $1,440, including farm rent, and set the benefit to $0 because 30% of net income is more than the maximum, never applying the minimum allotment. The reference's net income without farm rent is $1,269.19, which passes the net test and yields $23.84/month."
-us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Its bare $0 answer matches counting farm rent, which fails the net income test. The reference's countable income excludes farm rent, net income of $1,269.19 is below the $1,304 limit, and the $23.84/month one-person minimum allotment applies."
-us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"It set SNAP to $0 because 30% of net income is more than the maximum allotment, ignoring the minimum allotment and counting farm rent. The reference's net income is $1,269.19 without farm rent, which passes the net test, so the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"It found net income above the limit by counting all listed income, including farm rent. The reference's countable income leaves out farm rent, so net income is $1,269.19, below the $1,304.17 limit, and the household receives the $23.84/month minimum allotment."
-us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"It cited vehicle and asset resources, although Texas BBCE waives the asset test, and treated 30% of net income being more than the maximum as a zero benefit while counting farm rent. The reference's net income without farm rent is $1,269.19, which passes the net test, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Its $1,434 net income includes the $160/month of farm rent, which is above the net limit. The reference's countable income is $1,576.92 gross and $1,269.19 net without farm rent, which passes the net test, and the one-person minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent, used a gross limit of about $17,000 in place of Texas's 165% BBCE limit, and treated the $11,838 vehicle as disqualifying even though BBCE waives the asset test. The reference's gross income without farm rent is $18,923/year, net income of $1,269/month passes, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Its $0 answer rests on income and resources being too high, which matches counting farm rent and applying an asset test that Texas BBCE waives. The reference's countable income leaves out farm rent, net income of $1,269.19 passes, and the minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"It refused to compute SNAP, claiming the household facts lacked eligibility indicators, and defaulted to $0. The facts fully determine the result: gross income of $1,576.92, net income of $1,269.19 below the $1,304 limit, a $380.76 expected contribution above the $298 maximum, and therefore the $23.84/month one-person minimum allotment."
-us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"It counted the $1,920 of farm rent to reach $20,843 and failed a gross limit, without applying Texas's 165% BBCE threshold. The reference's countable income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"It failed the household on a gross limit after including farm rent, ignoring Texas's 165% BBCE limit. Without farm rent, gross income is 121% of the poverty guideline and net income of $1,269.19 passes the 100% test, so the one-person minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent, which pushed net income above the 100% limit. The reference's countable income excludes farm rent, so net income is $1,269.19 and below the $1,304.17 limit, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"It included farm rent and failed a gross income limit instead of applying Texas's 165% BBCE threshold. The reference's gross income is $1,576.92 without farm rent, net income of $1,269.19 passes the net test, and the household receives the $23.84/month minimum allotment."
-us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent in net income and found it above the 100% limit. Without farm rent, net income is $1,269.19, which passes the net test, so the one-person household receives the $23.84/month minimum allotment."
-us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Counting farm rent put its net income above the limit. The reference's countable income is wages plus financial assistance, giving net income of $1,269.19 below the $1,304.17 limit, and the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and concluded that income was too high for any benefit, never applying the one-person minimum allotment. Without farm rent, net income is $1,269.19, which passes the net test, and the household receives $23.84/month."
-us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Including farm rent put its net income too high for a benefit. The reference's net income without farm rent is $1,269.19, below the $1,304 limit, and because 30% of it is more than the $298 maximum, the one-person minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"It blamed income and assets for the $0, but Texas BBCE waives the asset test. Its answer matches counting farm rent, whereas the reference's net income without farm rent is $1,269.19, which passes the net test, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent together with the financial assistance and found the income limits exceeded. The reference's countable income leaves out farm rent, giving gross income of $1,576.92 (121% of the poverty guideline) and net income of $1,269.19 (97%), and pays the $23.84/month minimum allotment."
-us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and set SNAP to $0 because 30% of net income is more than the maximum, never applying the minimum allotment. The reference's net income is $1,269.19 without farm rent, which passes the net test, so the one-person minimum of $23.84/month is paid."
-us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent in its $20,843 income total, which put net income above the 100% limit. Without farm rent, net income is $1,269.19, below the $1,304.17 limit, and the $23.84/month minimum allotment applies."
-us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent to reach $20,843 and compared it with a 130% limit (about $20,700) instead of Texas's 165% BBCE limit. The reference's gross income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"It applied a 130% gross limit to income that includes farm rent and ignored Texas's 165% BBCE threshold. The reference's countable income without farm rent passes both tests (net income is $1,269.19), and the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $20,843 and applied the 130% gross limit, missing Texas's 165% BBCE limit. The reference's gross income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment is paid."
-us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"It denied categorical eligibility, even though Texas BBCE confers it through TANF non-cash assistance, and counted farm rent, which pushed net income too high. Without farm rent, net income is $1,269.19 and passes the net test, so the one-person minimum allotment of $23.84/month applies."
-us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"It claimed that a single adult with wage income does not qualify for SNAP; no such exclusion exists. The household passes the gross test (121% of the poverty guideline) and the net test ($1,269.19 against $1,304.17), and it receives the $23.84/month one-person minimum allotment."
-us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent to reach $1,737/month and failed a 130% limit of about $1,695, missing Texas's 165% BBCE limit. The reference's gross income is $1,576.92 without farm rent, net income of $1,269.19 passes, and the minimum allotment of $23.84/month is paid."
-us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"It correctly found that 30% of net income is more than the maximum, then threw that result out and substituted $3,576 by assuming a shelter deduction for which no housing costs were listed. With no shelter deduction, net income is $1,269.19, which is below the $1,304 limit, so the correct outcome is the one-person minimum allotment of $23.84/month, not a large benefit."
-us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Its bare finding of ineligibility matches counting farm rent, which fails the net test. The reference's countable income leaves out farm rent, net income of $1,269.19 passes the 100% test, and the one-person minimum allotment of $23.84/month applies."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $19,518. It put no state income tax into SALT ($4,064, where the engine's SALT is $9,431.59 with the engine's $5,367.30 Virginia withholding estimate), skipped the 0.5%-of-AGI charitable floor, and counted the $2,000 of OTC purchases, which the engine does not count, instead of the $2,434.80 Medicare Part B premium the engine imputes, so taxable income came to $78,077 rather than $72,807.33. It then computed about $11,889 at the correct 2026 brackets but replaced that with $8,971 through an arbitrary 'bracket inflation' adjustment that no rule supports."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its structure matches the engine's (itemize, $4,106 senior deduction, $4,872 QBI, 2026 brackets), but it put its own $3,524 Virginia tax into SALT ($7,588), where the engine counts a $5,367.30 Virginia withholding estimate ($9,431.59). It also built medical from the $2,000 of OTC purchases, which the engine does not count, while leaving out the $2,434.80 Medicare Part B premium the engine imputes ($4,007 versus the engine's $4,441.86). Together these put itemized deductions about $2,277 below the engine's, so taxable income rises to $75,085 and tax rises by about 22% of that gap."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,828.80 of taxable Social Security out of income, invented a $28,300 aged standard deduction, used 2024 brackets, and dropped the QBI and senior deductions. It then added $336 of Medicare payroll tax and an arbitrary $2,000 'safe harbor' adjustment to reach $10,268. Neither payroll tax nor withholding belongs in income tax before refundable credits; the engine's tax comes from $72,807.33 of taxable income after itemizing $24,787.23."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its intermediate work entered SALT at the $10,000 cap although it counted only the $4,064 real estate tax (the engine's SALT is $9,431.59, which includes the engine's $5,367.30 Virginia withholding estimate), dropped the 0.5%-of-AGI charitable floor, and took the full $6,000 senior deduction without the 6% phase-down on MAGI above $75,000 (correct amount $4,105.65), reaching about $10,711. It then submitted $5,489, a figure that matches none of the steps it computed."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed about $4,500 of mortgage interest from the loan balance although no interest was listed, so mortgage interest is $0. It also put no state income tax in SALT, where the engine counts a $5,367.30 Virginia withholding estimate, skipped the charitable floor, and omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. It computed $11,802 and then trimmed it to $11,210 with an unexplained 'rounding' adjustment instead of taxing the engine's $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of a self-employment tax (~$1,721) that does not exist here: the partnership/S-corp income carries no SE tax. It itemized only about $17,650, with no state income tax in SALT and medical from the $10,000 of other medical expenses alone, and chose the standard deduction with an unphased $6,000 senior deduction. The engine itemizes $24,787.23, counting a $5,367.30 Virginia withholding estimate in SALT and an imputed $2,434.80 Medicare Part B premium in medical, neither of which the prompt lists; without both, the engine also takes the standard deduction. The model then cut its own ~$11,900 estimate to $6,900 by citing nonrefundable credits this household does not have."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It followed the engine's structure but put its own $3,554.50 Virginia tax into SALT ($7,618.50), where the engine counts a $5,367.30 Virginia withholding estimate ($9,431.59). It also computed medical at $4,007 by counting the $2,000 of OTC purchases, which the engine does not count, instead of the $2,434.80 Medicare Part B premium the engine imputes (the engine's $4,441.86). Itemized deductions came out $2,247.63 below the engine's ($22,539.60 vs $24,787.23), and 22% of that gap matches the $494.49 by which the model's answer exceeds the engine's $10,729.61."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly set aside the OBBBA senior deduction ($4,105.65 after phase-down), imputed $6,693 of mortgage interest from the loan balance even though no interest was listed, put no state income tax in SALT, where the engine counts a $5,367.30 Virginia withholding estimate, and skipped the 0.5% charitable floor. It also used inflation-projected 2025 brackets (12% ending near $49,784) instead of the 2026 thresholds of $12,400 and $50,400, ending at $75,490 of taxable income instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction entirely. It itemized only $19,518: SALT limited to the $4,064 real estate tax (the engine's SALT is $9,431.59, which includes the engine's $5,367.30 Virginia withholding estimate), no 0.5% charitable floor, and $4,007 of medical that counts the $2,000 of OTC purchases, which the engine does not count, in place of the $2,434.80 Medicare Part B premium the engine imputes. That gave $82,183 of taxable income instead of $72,807.33. It also used guessed brackets (10% to $12,150, 12% to $49,400) instead of 2026's $12,400 and $50,400."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the senior deduction ($4,106), QBI ($4,872), and 2026 brackets correctly, but its itemized total of about $21,188 is $3,599 below the engine's $24,787.23. The engine's components are charitable $10,913.78, SALT $9,431.59 (including $5,367.30 that the engine estimates as Virginia income tax withheld), and medical $4,441.86 (including a $2,434.80 Medicare Part B premium the engine imputes). The resulting $76,407 of taxable income, instead of $72,807.33, adds about $792 of tax at 22%."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of a nonexistent self-employment tax ($1,721) from AGI, although the partnership/S-corp income carries no SE tax here. It also omitted the $4,105.65 senior deduction and itemized only $17,645: no state income tax in SALT, where the engine counts a $5,367.30 Virginia withholding estimate; no 0.5% charitable floor; and medical from the $10,000 of other medical expenses alone, where the engine also counts an imputed $2,434.80 Medicare Part B premium. The model's taxable income of $82,357 is about $9,550 above the engine's $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and skipped the 0.5%-of-AGI charitable floor. It also put its own $3,551.53 Virginia tax into SALT ($7,615.53), where the engine counts a $5,367.30 Virginia withholding estimate ($9,431.59), and held medical to $2,007.04, without the $2,434.80 Medicare Part B premium the engine imputes (the engine's $4,441.86). On top of that it used wrong brackets (10% to $12,200, 12% to $49,400 instead of $12,400 and $50,400), producing $80,630.83 of taxable income."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the pre-TCJA sunset schedule, with a $5,350 personal exemption and 10%/15%/25% brackets. OBBBA made the TCJA rate structure permanent, so for 2026 personal exemptions stay at zero and the brackets are 10%/12%/22%. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $79,372.69 of taxable income implies itemized deductions of about $22,328 plus QBI, with no senior deduction. It therefore omitted the $4,105.65 OBBBA senior deduction, which itemizers receive, and itemized about $2,460 less than the engine's $24,787.23, whose SALT includes the engine's $5,367.30 Virginia withholding estimate and whose medical deduction includes an imputed $2,434.80 Medicare Part B premium. The resulting $6,565 excess over the engine's $72,807.33 of taxable income drives the model's higher tax."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA law, with a $5,000 personal exemption and 10%/15%/25% brackets, even though OBBBA made the TCJA brackets permanent with no personal exemption. It also dropped the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Finally, it subtracted half of a nonexistent self-employment tax ($1,721), although the partnership/S-corp income carries no SE tax."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $110,094 of gross income counts 100% of the $23,328 of Social Security instead of the 85% maximum ($19,828.80) and ignores the $22 capital loss. Its roughly $81,300 of taxable income omits the $4,105.65 senior deduction, and the itemized deductions it lists (real estate taxes, charitable donations, medical expenses) include no state income tax; the engine itemizes $24,787.23, with a $5,367.30 Virginia withholding estimate in SALT and an imputed $2,434.80 Medicare Part B premium in medical. The model's $13,919 also exceeds the 2026-bracket tax on $81,300 (about $12,598), so it applied the wrong rate schedule as well."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax under 'pre-TCJA' 2026 law with personal exemptions, but OBBBA made the TCJA brackets and the zero personal exemption permanent. Its derivation also includes neither the $4,872.35 QBI deduction nor the $4,105.65 senior deduction. The engine taxes $72,807.33 at 10%/12%/22%, for $10,729.61."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA sunset, with a $5,050 personal exemption and the pre-TCJA 'sunset' brackets, although OBBBA made the TCJA rate schedule permanent with no personal exemption. It also omitted the $4,105.65 senior deduction and itemized only $19,256 versus the engine's $24,787.23; the engine's SALT alone is $9,431.59 because the engine adds a $5,367.30 Virginia withholding estimate to the $4,064.29 real estate tax."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The engine's computation is $106,572.56 AGI less $24,787.23 itemized, $4,872.35 QBI and $4,105.65 senior deductions, which leaves $72,807.33 taxed at $10,729.61. The model's $9,334 matches about $66,460 of taxable income at the 2026 brackets, so it overstated total deductions by roughly $6,350."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($6,000 reduced by 6% of MAGI over $75,000, to $4,105.65), which itemizers also claim. Its itemized total of $22,018.04 also falls $2,769 short of the engine's $24,787.23, whose SALT includes the engine's $5,367.30 Virginia withholding estimate and whose medical deduction includes an imputed $2,434.80 Medicare Part B premium. Together these leave the model's taxable income at $79,682.36, $6,875 above the engine's $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction instead of itemizing as the engine does: the engine's itemized deductions total $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59 with the engine's $5,367.30 Virginia withholding estimate, medical $4,441.86 with an imputed $2,434.80 Medicare Part B premium). The model also omitted the $4,105.65 senior deduction. Its $13,104.09 is about $11 above the $13,093.05 tax on $83,550.21 of taxable income (AGI less $18,150 and QBI only), which is $10,742.88 above the engine's $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted both the $4,872.35 QBI deduction and the $4,105.65 senior deduction. It also itemized only $17,518: SALT limited to the $4,064 real estate tax, where the engine counts a $5,367.30 Virginia withholding estimate ($9,431.59 in all); no 0.5% charitable floor; and $2,007 of medical from the $10,000 of other medical expenses alone, where the engine also counts an imputed $2,434.80 Medicare Part B premium. That puts taxable income near $89,055 instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted that QBI and the TCJA rates expired after 2025, so it applied a $5,500 personal exemption and 10%/15%/25% brackets. OBBBA made both the TCJA rate structure and the §199A deduction permanent, so the $4,872.35 QBI deduction applies, personal exemptions are zero, and the brackets are 10%/12%/22%. It also omitted the $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction. It also itemized only $17,518, with no state income tax in SALT ($4,064, where the engine counts a $5,367.30 Virginia withholding estimate for $9,431.59), no 0.5% charitable floor, and $2,007 of medical from the $10,000 of other medical expenses alone, where the engine also counts an imputed $2,434.80 Medicare Part B premium. The model's $84,183 of taxable income is $11,376 above the engine's $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and age-65 add-on wipe out all taxable income, ignoring the $106,572.56 of AGI from wages, partnership income, pension, IRA and 85% of Social Security. Even after $33,765.23 of deductions, $72,807.33 of taxable income remains and generates $10,729.61 at the 2026 brackets."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as living mainly on Social Security with modest taxable income. That overlooks $86,744 of wage, partnership, pension, IRA and interest income, which makes 85% of benefits taxable and puts AGI at $106,572.56. After $33,765.23 of total deductions, $72,807.33 remains taxable at rates up to 22%."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA sunset, with a personal exemption and the pre-2018 brackets, even though OBBBA made the TCJA brackets permanent and personal exemptions stay at zero in 2026. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Its itemized total was only about $17,518, which matches no state income tax in SALT, no 0.5% charitable floor and medical from the $10,000 of other medical expenses alone; the engine's $24,787.23 adds a $5,367.30 Virginia withholding estimate to SALT and an imputed $2,434.80 Medicare Part B premium to medical."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted half of a self-employment tax that does not exist, since the partnership/S-corp income carries no SE tax here, and it took the standard-deduction path instead of itemizing $24,787.23 as the engine does. Its $67,020 of taxable income is $5,787 below the engine's $72,807.33, so its total deductions exceeded the engine's even though it chose the path with smaller deductions in the engine."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction plus the $1,000 non-itemizer charitable deduction, with the senior and QBI deductions, for about $78,445 of taxable income, which reproduces the engine's standard-deduction result: $11,969.80 on $78,444.56. The engine gives that answer when it counts neither of two inputs the prompt does not list, a $5,367.30 Virginia withholding estimate in SALT and an imputed $2,434.80 Medicare Part B premium in medical expenses. With both, the engine itemizes $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59, medical $4,441.86), which exceeds the $19,150 and cuts tax by $1,240.19 to $10,729.61."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $11,540 matches about $76,490 of taxable income at the 2026 brackets, $3,684 above the engine's $72,807.33. The model's explanation gives no figures; if its other deductions match the engine's, its itemized deductions fell about $3,684 short of the engine's $24,787.23: charitable $10,913.78, SALT $9,431.59 (including the engine's $5,367.30 Virginia withholding estimate), and medical $4,441.86 (including a $2,434.80 Medicare Part B premium the engine imputes)."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the phased senior deduction ($4,105.63), QBI ($4,872.40), and 2026 brackets correctly but itemized only $20,639.33 after the charitable floor. That matches the engine's $20,639.28 with no Medicare Part B premium and with the engine's own Virginia tax on that reading ($3,654.15) in SALT, and the model's $11,642.20 matches the engine's $11,642.16 there. The reference instead counts a $2,434.80 Part B premium the engine imputes (medical $4,441.86) and puts the engine's $5,367.30 Virginia withholding estimate in SALT ($9,431.59), for $24,787.23; the $4,147.90 gap taxed at 22% ($912.54) accounts for the $912.59 by which the model's answer exceeds the reference."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. Its implied itemized deductions of about $21,026 also fall $3,761 short of the engine's $24,787.23, whose SALT includes the engine's $5,367.30 Virginia withholding estimate and whose medical deduction includes an imputed $2,434.80 Medicare Part B premium. Taxable income of $80,675 instead of $72,807.33 produces the model's $1,731 excess over the engine's tax at 22%."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It stated AGI as $96,572.80, about $10,000 below the engine's $106,572.56. That also inflated the senior deduction to $4,705.63, when the correct phase-down on $106,572.56 of MAGI gives $4,105.65. It then took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction instead of itemizing $24,787.23."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the phased senior deduction, the $4,872.40 QBI deduction, and the 2026 brackets correctly and gave no itemized figure. Its $11,642.09 matches the engine's $11,642.16 with no Medicare Part B premium and with the engine's own Virginia tax on that reading ($3,654.15) in SALT, where itemized deductions are $20,639.28. The reference's itemized total is $24,787.23: charitable $10,913.78, SALT $9,431.59 (including the engine's $5,367.30 Virginia withholding estimate), and medical $4,441.86 ($10,000 of other expenses plus a $2,434.80 Part B premium the engine imputes; the engine does not count OTC purchases). The model's answer exceeds the reference by $912.48, 22% of about $4,148."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The engine's computation reaches $72,807.33 of taxable income and $10,729.61 of tax, while the model's $10,234 matches about $70,555 of taxable income at the 2026 brackets. The model therefore claimed roughly $2,250 more deductions than the engine's $33,765.23 (itemized $24,787.23, QBI $4,872.35, senior $4,105.65)."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,468 personal exemption and 10%/15%/25% brackets, although OBBBA made the TCJA brackets permanent with no personal exemption for 2026. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA sunset took effect, using a $5,300 personal exemption and 10%/15%/25% brackets and declaring QBI expired. OBBBA made both the TCJA rate schedule and the §199A deduction permanent. As a result it dropped the $4,872.35 QBI deduction and the $4,105.65 senior deduction, and taxed $78,246 at the wrong rates."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction on the belief that it beats itemizing, but in the engine itemized deductions total $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59 with the engine's $5,367.30 Virginia withholding estimate, medical $4,441.86 with an imputed $2,434.80 Medicare Part B premium). Even on the standard path the model omitted the $1,000 non-itemizer charitable deduction, leaving $79,444.77 of taxable income instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. It also put its own $2,746 Virginia tax into SALT ($6,810), where the engine counts a $5,367.30 Virginia withholding estimate ($9,431.59), skipped the 0.5%-of-AGI charitable floor, and built its $4,007 of medical by counting the $2,000 of OTC purchases, which the engine does not count, in place of the $2,434.80 Medicare Part B premium the engine imputes. Taxable income of $79,437 instead of $72,807.33 drives its higher tax."
-us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets and a $5,300 personal exemption, although OBBBA made the TCJA brackets permanent and personal exemptions remain zero. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Finally, it cut AGI to $104,852 by subtracting half of a self-employment tax the partnership/S-corp income does not incur."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared itemized deductions of $17,518 with the $18,150 standard deduction and took the standard deduction plus the $1,000 non-itemizer charitable deduction, with the phased senior and QBI deductions. The $17,518 matches the $4,064 real estate tax with no state income tax in SALT, charitable gifts without the 0.5%-of-AGI floor and $2,007 of medical from the $10,000 of other medical expenses alone. The model's $11,969.85 matches the engine's $11,969.80 when the engine counts neither of two inputs the prompt does not list, a $5,367.30 Virginia withholding estimate in SALT and an imputed $2,434.80 Medicare Part B premium in medical expenses. With both, the engine itemizes $24,787.23 and gets $10,729.61, $1,240.19 less."
-us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It acknowledged that taxable income is positive and that itemizing beats the standard deduction, then reported $0 without identifying any credit. With AGI of $106,572.56 and total deductions of $33,765.23, taxable income is $72,807.33, which produces $10,729.61 at the 2026 brackets with no nonrefundable credit to offset it."
-us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated itemized deductions at about $19,500 and chose the standard deduction, but in the engine itemized deductions total $24,787.23 (charitable $10,913.78, SALT $9,431.59 including the engine's $5,367.30 Virginia withholding estimate, medical $4,441.86 including a $2,434.80 Medicare Part B premium the engine imputes). On its own standard path the model also omitted the $1,000 non-itemizer charitable deduction, leaving $79,445 of taxable income instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left Social Security out of income, although 85% ($19,828.80) is taxable. It also used an outdated $15,750 aged standard deduction and skipped both itemizing and the senior deduction. Its reported $15,654 is not even the tax on its own $66,122 of taxable income, which the 2026 brackets tax at about $9,258."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It stated AGI as $80,164 although its own listed items sum to $106,572.80, cut QBI to $3,550, and took a $20,400 standard deduction instead of itemizing $24,787.23. It then subtracted about $2,471 from its $7,761.45 tax by treating the QBI deduction and charitable amounts as nonrefundable credits, which they are not."
+us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added the $1,920 farm rent to countable income, which raised net income to about $1,429 and failed the 100% FPL net test. It also first applied a 130% gross test even though Texas BBCE covers the household. PolicyEngine's SNAP income is wages plus financial assistance: net is $1,269.19 (97% FPL), and the one-person minimum allotment of about $24/month ($287.68/yr) applies, not $0."
+us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Counted the $160/month of farm rent in SNAP income, so net income came out near $1,429 and failed the net test. It then let the formula go to zero because 30% of net exceeded $298. PolicyEngine excludes farm rent, so net is $1,269.19 (97% FPL), and an eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr) instead of $0."
+us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Applied the standard 130% FPL gross income limit and missed that Texas TANF-funded BBCE confers categorical eligibility with a 165% FPL gross limit. The household's gross income of $1,576.90 (121% FPL) is within that limit, net income passes at 97% FPL, and the household gets the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Declared the household not established as eligible without running any income test. The household passes Texas's BBCE gross test (121% FPL) and the net test ($1,269.19, 97% FPL), so it is eligible. Because 30% of net exceeds the $298 maximum, it receives the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Disqualified the household with a 130% FPL gross income test and ignored Texas BBCE through TANF non-cash assistance, which raises the gross limit to 165% FPL. Gross income passes at 121% FPL and net passes at 97% FPL, so the household gets the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Treated the $13,000 financial assistance as an excluded gift and counted only wages and farm rent, which made net income low and produced a large benefit ($2,304). PolicyEngine counts the financial assistance as unearned income, so net income is $1,269.19. Thirty percent of that ($380.76) exceeds the $298 maximum, leaving only the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Left out the $13,000 financial assistance from countable income and assumed a near-maximum benefit of about $275/month. With the assistance counted as unearned income, net income is $1,269.19 and the expected contribution of $380.76 exceeds the $298 maximum. The household therefore receives only the minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly invoked Texas BBCE but assumed the $13,000 financial assistance does not count, which gave $346 of net income and a $194/month benefit. PolicyEngine counts the assistance ($1,083.33/mo) as unearned income, so net income is $1,269.19. The 30% contribution then exceeds the $298 maximum, and only the one-person minimum allotment ($287.68/yr) remains."
+us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Applied a 130% FPL gross income limit of about $1,632/month and concluded the household was ineligible, missing Texas TANF-funded BBCE (165% FPL gross limit). Gross income of $1,576.90 (121% FPL) passes, net income of $1,269.19 passes the 100% FPL test, and the one-person minimum allotment applies ($287.68/yr)."
+us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Used the 130% FPL gross income threshold to deny eligibility and never applied Texas BBCE, which confers categorical eligibility up to 165% FPL. The household passes at 121% FPL gross and 97% FPL net, and receives the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $13,000 financial assistance from countable income and computed $194/month. PolicyEngine counts it as unearned income, so net income is $1,269.19 and 30% of net ($380.76) exceeds the $298 maximum. The household gets only the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Failed the household on the 130% FPL gross income test and ignored Texas BBCE, which waives that test up to 165% FPL through TANF non-cash assistance. The household is categorically eligible, passes the net test at 97% FPL, and gets the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross limit (about $1,672/month) and stopped there, missing Texas BBCE's 165% FPL gross limit. SNAP gross income of $1,576.90 (121% FPL) passes and net income of $1,269.19 passes, so the minimum one-person allotment of about $24/month ($287.68/yr) is paid."
+us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum allotment but then set the benefit to $0. Eligible one- and two-person households receive the minimum allotment (8% of the $298 maximum, about $24/month), which gives $287.68 for the year."
+us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Computed a garbled net income ($6,454 with a $1,936 contribution) and zeroed the benefit because the contribution exceeded the maximum allotment. The household's monthly net income is $1,269.19, which passes the net test. Because it is an eligible one-person household, it gets the minimum allotment of about $24/month ($287.68/yr), not $0."
+us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Included the $1,920 farm rent in SNAP income, which put net income at $1,440.20, and then failed the household on the net income test. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. The categorically eligible one-person household gets the minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Asserted ineligibility without any computation. The household is categorically eligible through Texas BBCE (gross 121% FPL), passes the net test at 97% FPL, and receives the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit because 30% of net income exceeds the one-person maximum and never applied the SNAP minimum benefit. An eligible one-person household whose formula benefit falls below the floor receives the minimum allotment of about $24/month, which totals $287.68 for the year."
+us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, which put net income at $1,440, and reduced the benefit to $0 because 30% of net exceeded the maximum allotment. That skips the minimum allotment for eligible one-person households. PolicyEngine's net income is $1,269.19 (eligible), and the floor of about $24/month yields $287.68/yr."
+us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Reported $0 with no derivation, which matches letting the benefit formula go to zero. The household is eligible: gross 121% FPL under Texas BBCE and net 97% FPL. Because 30% of net exceeds the $298 maximum, the one-person minimum allotment of about $24/month ($287.68/yr) applies."
+us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Concluded $0 because 30% of net income exceeds the one-person maximum allotment and omitted the minimum benefit rule. Eligible one-person households receive at least 8% of the maximum allotment, about $24/month, for $287.68 a year."
+us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Claimed net income exceeds the limit, which only holds if farm rent is counted. PolicyEngine's SNAP income is wages plus financial assistance, giving net income of $1,269.19 (97% FPL). The household passes and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit because 30% of net income exceeds the maximum allotment, and also cited the vehicle and assets even though the asset test is met ($10 bank account). The eligible one-person household receives the minimum allotment of about $24/month, $287.68 for the year."
+us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 farm rent, which put net income at about $1,434, above 100% FPL, and declared the household ineligible. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. The one-person minimum allotment then yields $287.68/yr."
+us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Used a 130% FPL gross limit understated at about $17,000 and also treated the $11,838 vehicle as breaking the asset limit. Texas BBCE raises the gross limit to 165% FPL, and the household passes the asset test with $10 in the bank. The household is therefore eligible and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and resources are too high with no supporting computation. The household passes the Texas BBCE gross test (121% FPL), the net test (97% FPL), and the asset test ($10), so it is eligible. It receives the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 on the claim that the facts do not support a benefit computation, even though wages, financial assistance, and household size fully determine SNAP. The single-person household is eligible at 97% FPL net income and receives the minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross income limit and ignored Texas TANF-funded BBCE, which sets a 165% FPL gross limit. The household is categorically eligible, passes the net test at 97% FPL, and gets the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Failed the household on the standard gross-income limit without applying Texas BBCE, under which gross income of 121% FPL passes the 165% FPL limit. With net income at 97% FPL, the household is eligible and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and concluded the household fails the net income test. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL), below the $1,304.17 limit. The household is eligible, and its benefit is the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Applied the ordinary one-person gross income limit and missed Texas BBCE's 165% FPL threshold through TANF non-cash assistance. Gross income passes at 121% FPL and net income passes at 97% FPL, so the minimum one-person allotment ($287.68/yr) is paid."
+us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Included farm rent in countable income and concluded that net income exceeds the 100% FPL limit. PolicyEngine counts only wages and financial assistance, so net income is $1,269.19 (97% FPL). The household is eligible and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent alongside wages and financial assistance, which pushed net income above the limit. PolicyEngine excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. Thirty percent of net exceeds the $298 maximum, so the one-person minimum allotment of about $24/month ($287.68/yr) applies."
+us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"Concluded that income is too high to produce a positive benefit and omitted the minimum allotment rule. The household is eligible at 97% FPL net income, and although the formula benefit is negative, eligible one-person households get about $24/month ($287.68/yr)."
+us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Treated the formula's non-positive result as no benefit and never applied the SNAP minimum allotment for one-person households. The household is eligible (net $1,269.19, 97% FPL), so it receives about $24/month, $287.68 for the year."
+us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and assets exceed SNAP limits without computing either. Assets are $10, gross income is 121% FPL (within Texas BBCE's 165% limit), and net income is 97% FPL, so the household is eligible for the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Declared ineligibility on the gross or net test without applying Texas BBCE, under which gross income of 121% FPL passes, while net income of $1,269.19 is 97% FPL. The eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum allotment but set SNAP to $0 instead of applying the minimum allotment. Eligible one-person households receive 8% of the maximum, about $24/month, totaling $287.68."
+us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 farm rent in SNAP income, so net income landed above 100% FPL and the household was found ineligible. PolicyEngine's SNAP income excludes farm rent, giving net income of $1,269.19 (97% FPL). The household is eligible and receives the minimum one-person allotment ($287.68/yr)."
+us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Compared gross income against a 130% FPL limit (about $20,700) and missed Texas BBCE, whose 165% FPL gross limit the household easily meets. Net income passes at 97% FPL, so the household receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross income limit and ignored Texas TANF non-cash BBCE, which waives that test up to 165% FPL. The household is categorically eligible, passes the net test at 97% FPL, and receives the minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"Required gross income at or below 130% FPL and never applied Texas BBCE, which confers categorical eligibility with a 165% FPL gross limit. Gross income is 121% FPL and net income is 97% FPL, so the one-person minimum allotment ($287.68/yr) applies."
+us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Stated that the household lacks TANF/SSI categorical eligibility, but Texas BBCE confers categorical eligibility through TANF non-cash assistance. It also let the expected contribution eliminate the allotment. An eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr) even when 30% of net income exceeds the $298 maximum."
+us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed that a single wage-earning adult does not qualify for SNAP, but one-person households are eligible under the standard income tests. This household passes Texas BBCE (gross 121% FPL) and the net test (97% FPL), and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Failed the household on a 130% FPL gross limit of about $1,695/month and missed Texas BBCE's 165% FPL gross limit. The household is categorically eligible, passes the net test at $1,269.19 (97% FPL), and receives the one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment, then dropped that result and submitted $3,576, the full $298/month maximum, by invoking an unlisted shelter deduction. With no shelter costs, the benefit formula is negative, and the eligible one-person household receives only the minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible with no computation. It is categorically eligible through Texas BBCE (gross 121% FPL), passes the net test (97% FPL), and receives the one-person minimum allotment of about $24/month ($287.68/yr)."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $19,518. That total left Virginia income tax out of SALT, counted the $2,000 of OTC expenses but not the Medicare Part B premium in medical, and skipped the 0.5% charitable floor. Those choices gave taxable income of $78,077 and tax of about $11,889. It then cut that to $8,971 with an unexplained 'estimation' step that its own computation does not support."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"The structure was correct (itemize, senior deduction $4,106, QBI $4,872, 2026 brackets), but itemized deductions came to only $22,510. It estimated Virginia income tax at $3,524 instead of about $5,367, and it counted OTC expenses but not the $2,434.80 Medicare Part B premium in medical. That left taxable income $2,277 too high, at $75,085."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income and used an invented $28,300 standard deduction. It also refused to itemize, which contradicts its own comparison. It then added $336 of Medicare payroll tax to income tax and an arbitrary $2,000 'safe harbor' adjustment, so $10,268 does not come from any valid tax computation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed taxable income falls in a zero-tax range. This ignored about $106,573 of AGI ($86,744 of non-Social Security income plus 85% of benefits) that leaves $72,807 of taxable income after all deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It entered SALT as the full $10,000 cap when only $4,064 of real estate tax was listed. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 MAGI and left out the Medicare Part B premium from medical. After reaching $10,711, it submitted $5,489 with no supporting derivation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $4,500 of mortgage interest even though no interest was listed. It left Virginia income tax out of SALT and the Part B premium out of medical, and it omitted the OBBBA senior deduction ($4,105.65) entirely. It computed $11,802, then lowered it to $11,210 with an unexplained adjustment."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a nonexistent half-SE-tax deduction of $1,721 for the partnership income. It took the standard deduction plus an unphased $6,000 senior bonus instead of itemizing $24,787. Its own math reached about $11,900, and it then submitted $6,900 with no supporting computation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The method was correct, but SALT was understated at $7,618.50 (Virginia income tax estimated at about $3,554 instead of about $5,367). Medical was $4,007 because it counted OTC expenses and not the $2,434.80 Part B premium. Itemized deductions were about $2,248 short, which left taxable income at $75,055 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $6,693 of mortgage interest from the loan balance although no interest was listed. It left out Virginia income tax and the Part B premium, skipped the charitable floor, and omitted the OBBBA senior deduction. It also used self-inflated 2025 brackets instead of the 2026 schedule."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 phased senior deduction. It also itemized only $19,518, with no Virginia income tax in SALT and OTC counted instead of the Part B premium in medical. That produced taxable income of $82,183 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only about $21,188 instead of $24,787. It underestimated Virginia income tax in SALT and left the $2,434.80 Medicare Part B premium out of medical. Taxable income ended about $3,600 too high, at $76,407."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $1,721 of half-SE-tax that does not apply to this partnership/S-corp income. It omitted the senior deduction and left Virginia income tax and the Part B premium out of itemized deductions. Taxable income came to $82,357."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65) and counted only $10,000 of medical, missing the $2,434.80 Part B premium. It also understated Virginia income tax at $3,551.53. Taxable income came to $80,631 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired pre-TCJA regime for 2026: a $5,350 personal exemption and 10/15/25% brackets, with no QBI deduction and no senior deduction. OBBBA made the TCJA rate structure permanent and added the senior deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $79,373 is about $6,565 above the correct $72,807. That gap matches omitting the $4,105.65 phased senior deduction and understating itemized deductions (Virginia income tax and the Medicare Part B premium in medical)."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used a pre-TCJA $5,000 personal exemption and 10/15/25% brackets instead of the permanent OBBBA 2026 brackets. It left out the QBI and senior deductions and also subtracted a nonexistent $1,721 half-SE-tax adjustment."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated gross income at $110,094 instead of AGI of $106,573 and omitted the senior deduction. Taxable income came to about $81,300. Its $13,919 is also more than the 2026 10/12/22% brackets produce on that amount, so its rates were wrong as well."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly applied 2026 pre-TCJA law with personal exemptions and sunset brackets. That leaves out the QBI deduction, the OBBBA senior deduction, and the permanent 10/12/22% rate schedule."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used 'sunset' brackets and a $5,050 personal exemption, which no longer exist after OBBBA. It omitted the $4,105.65 senior deduction and itemized only $19,256, without the Virginia income tax and the Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The correct path yields taxable income of $72,807.33 and tax of $10,729.61. Its $9,334 corresponds to taxable income of about $66,460 under the 2026 brackets, so it overstated deductions by roughly $6,300."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65). It itemized only $22,018 because it underestimated Virginia income tax and left out the Medicare Part B premium, which left taxable income at $79,682."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction even though itemized deductions of $24,787 (SALT with Virginia income tax, medical with the Part B premium, charity) are larger. It also left out the phased senior deduction, so taxable income came to about $83,550."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $4,872 QBI deduction or the $4,105.65 senior deduction. It itemized only $17,518, with no Virginia income tax in SALT and no Part B premium in medical, which put taxable income near $89,055."
+us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted that the QBI deduction lapsed after 2025 and applied a personal exemption with 10/15/25% brackets. OBBBA made both the QBI deduction and the TCJA rate schedule permanent and added the senior deduction, which it also omitted."
+us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction. It itemized only $17,518, with no Virginia income tax in SALT and no Part B premium or OTC in medical. Taxable income came to $84,183 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard and age-65 deductions offset all income. That ignores about $106,573 of AGI, which leaves $72,807 of taxable income even after itemized, QBI and senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the income was mainly Social Security with modest taxable income. It ignored $86,744 of wages, partnership, pension and IRA income, which also makes 85% of benefits taxable."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-2025' pre-TCJA brackets and a personal exemption instead of the permanent OBBBA 10/12/22% schedule. It also omitted the QBI and senior deductions and itemized only $17,518."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of an imputed self-employment tax that does not apply to this partnership/S-corp income. It also took the standard-deduction path, and its overstated deductions gave taxable income of $67,020 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction ($19,150 total). Itemized deductions of $24,787 are larger and should have been used. That choice left taxable income at $78,445 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $11,540 implies taxable income of about $76,490, roughly $3,680 above the correct amount. That matches understated itemized deductions from leaving out the full Virginia income tax in SALT and the Medicare Part B premium in medical."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"The senior deduction ($4,105.63), QBI deduction and brackets were all correct, but itemized deductions were only $20,639 instead of $24,787. It underestimated Virginia income tax in SALT and left the $2,434.80 Part B premium out of medical."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $80,675 is about $7,870 too high. It omitted the $4,105.65 senior deduction and understated itemized deductions by leaving out the full Virginia income tax and the Medicare Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped $10,000 from AGI ($96,572.80 instead of $106,572.80), which inflated the senior deduction to $4,705.63. It also took the standard plus $1,000 non-itemizer charitable deduction instead of itemizing $24,787."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The senior deduction, QBI deduction and 2026 brackets were correct, but itemized deductions were about $4,148 too low. It underestimated imputed Virginia income tax in SALT and excluded the $2,434.80 Medicare Part B premium from medical."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The correct taxable income is $72,807.33. Its $10,234 corresponds to taxable income of about $70,555 under the 2026 brackets, so it overstated deductions by roughly $2,250."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law: a $5,468 personal exemption, restored 10/15/25% brackets, and no QBI or senior deduction. OBBBA kept the 10/12/22% schedule and the QBI deduction permanently and added the senior deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset and declared the QBI deduction expired. It used a $5,300 personal exemption and 10/15/25% brackets and left out the OBBBA senior deduction, none of which matches 2026 law."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded the $18,150 standard deduction beats itemizing. Itemized deductions are $24,787 once SALT includes Virginia income tax and medical includes the Part B premium, so taxable income should have been $72,807 instead of $79,445."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65). It understated itemized deductions by estimating Virginia income tax at $2,746 and leaving out the Part B premium, which left taxable income at $79,437."
+us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets and a $5,300 personal exemption instead of the permanent OBBBA schedule. It omitted the QBI and senior deductions and subtracted a nonexistent half-SE-tax adjustment from AGI."
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed itemized deductions as only $17,518, leaving Virginia income tax out of SALT and the Part B premium out of medical. As a result it took the standard plus non-itemizer charitable deduction instead of itemizing $24,787, and taxable income came to $78,445."
+us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It acknowledged positive taxable income but still reported $0 tax with no credit to justify it. It also invented mortgage interest, while about $72,807 of taxable income at 2026 brackets produces $10,729.61."
+us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated itemized deductions at about $19,500, leaving Virginia income tax out of SALT and the Medicare Part B premium out of medical. It therefore chose the standard deduction, which put taxable income at $79,445 instead of itemizing $24,787."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left all $19,829 of taxable Social Security out of income and used an invented $15,750 standard deduction. It never considered itemizing, and its reported $15,654 is far above what the 2026 brackets produce even on its own $66,122 taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It reported AGI as $80,164, which does not match its own listed components that sum to $106,573. It cut the QBI deduction to $3,550 and took a $20,400 standard deduction instead of itemizing. It then subtracted invented reductions that treat deductions as credits, going from $7,761 to $5,290."
us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295."
us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract."
us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component."
@@ -8909,52 +9130,53 @@ us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"Tr
us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no self_employment_tax value and no explanation, so there is no substantive calculation to assess. The correct output is $0 because the household has no net earnings from self-employment."
us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as SE income and computed $3,442.62. It then submitted an unexplained $3,459.40. The pass-through income is not net earnings from self-employment, so SE tax is $0."
us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as net SE income, which is the core error because it is not SE earnings, so SE tax is $0. It also added a made-up 'additional Medicare-equivalent portion above the Social Security wage base'. No income here comes near that base, and this pushed its answer to $3,486.63."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its own path got taxable income of $66,296 and tax of about $3,555, but it then submitted $2,597, an unexplained cut of about $958 with no rule behind it. That path also left out the $800 age-65 exemption and put itemized deductions at $19,518 instead of the engine's $19,419.94; with the engine's figures, taxable income is $65,593.83 and tax $3,514.15."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the age deduction from the $75,000 married-joint threshold and kept $256. For a single filer the threshold is $50,000, and AFAGI of $86,744 wipes the $12,000 deduction out entirely. Its $18,985 of itemized deductions applies the 0.5%-of-AGI charitable floor, as the engine does, but its federal explanation builds medical from $12,000, counting the $2,000 of over-the-counter expenses, which the engine leaves out, in place of the $2,434.80 Medicare Part B premium the engine imputes ($4,007 against $4,441.86). That leaves it $435 below the engine's $19,419.94, so the model's taxable income of $65,773 ends up $179 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It shrank $86,744 of non-Social Security income to an 'AGI' of $27,752 and applied a $28,000 federal-style standard deduction. It then claimed zero tax, even though Virginia's 2% bracket starts at the first dollar. Its base should be VA AGI of $86,743.77; the engine then subtracts $19,419.94 of itemized deductions and $1,730 of exemptions, which leaves $65,593.83 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning got $4,227, but it submitted $1,854, which nothing in its derivation supports. Even the $4,227 path is wrong: it used an $8,750 standard deduction instead of the engine's $19,419.94 of Virginia itemized deductions (in the engine the filer itemizes federally), and it left out the $930 personal and $800 age-65 exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax with the $8,500 standard deduction and only the $930 exemption, then vaguely 'adjusted for itemizing' to $4,014. That figure implies taxable income of about $74,300, meaning total deductions of only about $12,450. The engine deducts $21,149.94 ($19,419.94 itemized plus $1,730 exemptions); the model never computed the itemized total and dropped the $800 aged exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,500 standard deduction instead of Virginia itemized deductions of $19,419.94, which gave taxable income of about $76,500. It then took about $281 off the tax as 'personal exemption credits'. Virginia's $930 personal and $800 aged exemptions are subtractions from taxable income, not credits."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It took only the $930 personal exemption and left out the $800 Virginia exemption for a filer 65 or older. It also used $19,518 of itemized deductions instead of $19,419.94. Together these put taxable income at $66,296 instead of $65,593.83, so its tax is about $40 high."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $9,000 standard deduction instead of the engine's $19,419.94 of Virginia itemized deductions (in the engine the filer itemizes federally). It then subtracted the $930 personal and $800 aged exemptions dollar-for-dollar from tax as credits. They actually reduce taxable income, where they are worth only about $99 at the 5.75% rate."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed the full $12,000 age deduction, but for a single filer it drops $1 for each $1 of AFAGI above $50,000, so AFAGI of $86,744 eliminates it. It also left out the $1,730 in exemptions and then cut its own $2,947.50 result to $2,450 with no supporting step."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions of about $17,518 match charitable gifts without the 0.5%-of-AGI floor ($11,446.65), the $4,064.29 real estate tax and $2,007.06 of medical from the $10,000 of other medical expenses alone. The engine's $19,419.94 applies the floor ($10,913.78) and adds to medical a $2,434.80 Medicare Part B premium the engine imputes ($4,441.86); like the model, the engine leaves out the $2,000 of over-the-counter health expenses. The model also took only the $930 exemption without the $800 age-65 exemption, so its taxable income came out at $68,296 instead of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It understated federal AGI at $104,874 instead of $106,573. It also applied a roughly $4,800 standard deduction plus an $800 'aged addition' instead of the $19,419.94 Virginia itemized deductions, which pushed taxable income to $78,515 instead of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the full $12,000 age deduction, but single-filer AFAGI of $86,744 is more than $12,000 over the $50,000 threshold, so the deduction is zero. It also used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and left out the $1,730 in exemptions. Its $66,244 taxable income is close to the engine's $65,593.83 only because these errors offset."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Every step matches the correct derivation (Social Security subtraction, zero age deduction, $1,730 exemptions, brackets) except the itemized total. It used $19,518.04 of Virginia itemized deductions, $98.10 more than the engine's $19,419.94, which lowers taxable income to $65,495.96 instead of $65,593.83 and tax by $5.63."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It computed Virginia taxable income from full federal AGI of $106,572.80 without subtracting the $19,828.80 of federally taxable Social Security, which Virginia exempts. It also used $17,518 of itemized deductions instead of $19,419.94 and left out the $800 age-65 exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated itemized deductions at $23,948.60, which is $4,528.66 more than the engine's $19,419.94. It took only the $800 aged exemption and dropped the $930 personal exemption. Its stated taxable income of $60,294.40 does not even match its own inputs ($86,744 − $23,948.60 − $800 = $61,995.40)."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,180 implies taxable income of about $77,170. That matches VA AGI of $86,744 minus a roughly $8,500–8,750 standard deduction and a single $930 exemption. The engine deducts $19,419.94 of Virginia itemized deductions plus $1,730 of personal and age exemptions, which leave $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation gives no figures. Its $3,726 implies taxable income of about $69,278, so about $17,466 came off Virginia AGI of $86,743.77, where the engine takes off $21,149.94: $19,419.94 of itemized deductions, whose $4,441.86 medical deduction adds a $2,434.80 Medicare Part B premium the engine imputes, and $1,730 of exemptions. One reconstruction close to its answer is $17,518 of itemized deductions (charitable gifts without the 0.5%-of-AGI floor, the real estate tax and medical from the $10,000 of other medical expenses alone) with no exemptions, which gives $3,722.98, about $3 below its answer."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $15,511 of itemized deductions is charitable gifts without the 0.5%-of-AGI floor plus real estate taxes; it leaves out the medical expense deduction. In the engine that deduction is $4,441.86: the $10,000 of other medical expenses plus a $2,434.80 Medicare Part B premium the engine imputes, less 7.5% of federal AGI ($7,992.94); the engine leaves out the $2,000 of over-the-counter expenses and applies the charitable floor, for $19,419.94 in all. The model also used a $1,600 exemption instead of Virginia's $930 personal plus $800 age-65 exemptions ($1,730)."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $5,617 implies taxable income of about $102,200. That is federal AGI of $106,573 with no $19,828.80 Social Security subtraction and only about a $4,400 deduction. The engine subtracts the Social Security, then $19,419.94 of itemized deductions and $1,730 of exemptions, leaving $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518.04 of itemized deductions matches charitable gifts without the 0.5%-of-AGI floor, the real estate tax and medical from the $10,000 of other medical expenses alone. The engine's $19,419.94 applies the floor ($10,913.78 rather than $11,446.65) and adds to medical a $2,434.80 Medicare Part B premium the engine imputes ($4,441.86 rather than $2,007.06); like the model, the engine leaves out the $2,000 of over-the-counter health expenses. The model's taxable income came out at $67,495.96 instead of $65,593.83, so its tax is $109 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation gives no figures, but its $3,623.52 matches $17,518 of itemized deductions with the $1,730 of exemptions: charitable gifts without the 0.5%-of-AGI floor, the real estate tax and medical from the $10,000 of other medical expenses alone. The engine's $19,419.94 applies the floor and adds to medical a $2,434.80 Medicare Part B premium the engine imputes; both totals leave out the $2,000 of over-the-counter health expenses. The model's implied taxable income is $1,902 above the engine's $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used $17,518 of itemized deductions, which its federal explanation breaks into $11,447 of charitable gifts with no 0.5%-of-AGI floor, $4,064 of real estate taxes and $2,007 of medical from the $10,000 of other medical expenses. The engine's $19,419.94 Virginia itemized deduction applies the floor ($10,913.78) and adds to medical a $2,434.80 Medicare Part B premium the engine imputes ($4,441.86); like the model, the engine leaves out the $2,000 of over-the-counter health expenses. That makes the model's taxable income $67,496 instead of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its described path ($106,573 − $19,829 − $9,000 standard deduction − $930) gives about $76,800 of taxable income and about $4,200 of tax. It instead submitted $14,036.62, which implies roughly $248,000 of taxable income, far more than the household's total income. The path itself also uses the standard deduction instead of the engine's $19,419.94 of itemized deductions and invents a pension deduction."
-us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,500 standard deduction instead of $19,419.94 of Virginia itemized deductions and counted a $930 age exemption instead of $800. It also made arithmetic errors: $106,573 − $19,829 comes out as $87,744 instead of $86,744. It then computed tax as $270 plus 5.75% of the amount over $8,000 instead of $720 plus 5.75% of the amount over $17,000."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions shelter all income. After the Social Security subtraction, VA AGI is still $86,743.77, and $19,419.94 of itemized deductions plus $1,730 of exemptions leave $65,593.83 taxable, which produces $3,514.15 of tax."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated retirement income as largely sheltered, but Virginia subtracts only Social Security. The $30,000 pension and $9,600 IRA distributions stay fully taxable, and the $12,000 age deduction is fully phased out at AFAGI of $86,744. That leaves $65,593.83 of taxable income after itemized deductions and exemptions, not $0."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used $17,518 of itemized deductions, which matches charitable gifts without the 0.5%-of-AGI floor, the real estate tax and medical from the $10,000 of other medical expenses alone. The engine's $19,419.94 Virginia itemized deduction applies the floor and adds to medical a $2,434.80 Medicare Part B premium the engine imputes; like the model, the engine leaves out the $2,000 of over-the-counter health expenses. So the model's taxable income of $67,496 is $1,902 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $12,000 age deduction as 'available'. For a single filer it is reduced $1 for $1 above $50,000 of AFAGI, so $86,744 of AFAGI eliminates it. That pushed taxable income down to about $51,647 instead of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,750 standard deduction and only the $930 personal exemption, leaving out the $800 age-65 exemption. When the engine counts neither of two inputs the prompt does not list, a $5,367.30 Virginia withholding estimate in federal SALT and an imputed $2,434.80 Medicare Part B premium, it also takes the federal standard deduction (the result the model's federal answer matches) and the $8,750 Virginia standard deduction, for $76,263.77 of taxable income and $4,127.67 of tax; the model's $77,064 is $800 higher there. With both inputs the filer itemizes federally in the engine, and the engine's $19,419.94 of Virginia itemized deductions and $1,730 of exemptions leave $65,593.83, $11,470 below the model's."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,658 implies taxable income of about $85,490. That is what results from taking about $21,000 of itemized deductions and exemptions from federal AGI of $106,573 without first subtracting the $19,828.80 of taxable Social Security, which Virginia exempts."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the Medicare Part B premium: charitable gifts after the 0.5%-of-AGI floor, as in the engine, the real estate tax and medical from the $10,000 of other medical expenses alone. With the $2,434.80 premium the engine imputes, the engine's Virginia itemized deduction is $19,419.94, so the model's taxable income of $68,028.82 is $2,435 above the engine's, and the model's $3,654.16 matches the engine's $3,654.15 without the premium."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 implies taxable income of about $63,560. That is about $2,037 below the engine's $65,593.83. The model's explanation describes the Social Security subtraction and the age and personal exemptions, so the gap points to itemized deductions above the engine's $19,419.94."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its about $57,229 of taxable income matches carrying into Virginia its own federal AGI of $96,572.80, which its federal answer states, about $10,000 below the engine's $106,572.56: Virginia AGI of $76,744, less about $17,785 of itemized deductions and $1,730 of exemptions, is about $57,229. Those itemized deductions are charitable gifts after a 0.5% floor on its AGI (about $10,964), the $4,064 real estate tax and about $2,757 of medical, the $10,000 of other medical expenses above 7.5% of its AGI. The engine deducts $19,419.94 of itemized deductions from Virginia AGI of $86,743.77, with a $4,441.86 medical deduction that adds a $2,434.80 Medicare Part B premium the engine imputes. Virginia's rates on $57,229 give about $3,033, $10 below the model's submitted $3,043.16."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the Medicare Part B premium: charitable gifts after the 0.5%-of-AGI floor, as in the engine, the real estate tax and medical from the $10,000 of other medical expenses alone. With the $2,434.80 premium the engine imputes, the engine's Virginia itemized deduction is $19,419.94, so the model's taxable income is $68,028.82 instead of $65,593.83. Virginia's rates on $68,028.82 give $3,654.16, within a cent of the engine's $3,654.15 without the premium; the model's $3,654.66 adds a $0.50 arithmetic slip."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted the standard deduction and exemptions offset all tax. After the Social Security subtraction, VA AGI is still $86,743.77, and deductions of $19,419.94 itemized plus $1,730 exemptions leave $65,593.83 taxable, which yields $3,514.15."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used $17,518 of Virginia itemized deductions, which matches charitable gifts without the 0.5%-of-AGI floor, the real estate tax and medical from the $10,000 of other medical expenses alone, instead of the engine's $19,419.94. The engine applies the floor and adds to medical an imputed $2,434.80 Medicare Part B premium; like the model, the engine leaves out the $2,000 of over-the-counter health expenses. The model's taxable income of $67,496 is $1,902 above the engine's $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Everything matches the correct derivation except the itemized total. It used $19,518 of itemized deductions, $98 more than the $19,419.94 Virginia itemized deduction, which lowers taxable income to $65,496 instead of $65,593.83 and tax by about $5."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer took the federal standard deduction and so used an $8,500 Virginia standard deduction, with the $1,730 of exemptions, for $76,514 of taxable income. When the engine counts neither of two inputs the prompt does not list, a $5,367.30 Virginia withholding estimate in federal SALT and an imputed $2,434.80 Medicare Part B premium, it also takes the federal standard deduction and the $8,750 Virginia standard deduction, for $76,263.77 of taxable income, $250 below the model's. With both inputs the filer itemizes federally in the engine, and the engine's $19,419.94 of Virginia itemized deductions leave $65,593.83, $10,920 below the model's."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It taxed income above $17,000 at 5.3% instead of Virginia's 5.75% top rate. It also subtracted the $12,000 age deduction, which is fully phased out at a single filer's AFAGI of $86,744, and left out the $1,730 of personal and aged exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Its own inputs ($86,744 − $19,647 itemized − $930) give $66,167, yet it taxed $64,446, which is $1,148 below the engine's $65,593.83. It also left out the $800 age-65 exemption and overstated itemized deductions compared with the engine's $19,419.94."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It took an $8,500 Virginia standard deduction, to match its federal standard deduction, and counted a $930 age-65 exemption instead of $800, which put taxable income at $76,384. When the engine counts neither of two inputs the prompt does not list, a $5,367.30 Virginia withholding estimate in federal SALT and an imputed $2,434.80 Medicare Part B premium, it also takes the federal standard deduction (the result the model's federal answer matches) and an $8,750 Virginia standard deduction, for $76,263.77 of taxable income and $4,127.67 of tax, $120.23 and $6.91 below the model's. With both inputs the filer itemizes federally in the engine, and the engine's $19,419.94 of Virginia itemized deductions leave $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as a low-income retiree with about zero tax, but VA AGI after the Social Security subtraction is $86,743.77. In the engine only $19,419.94 of itemized deductions and $1,730 of exemptions apply, leaving $65,593.83 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It computed Virginia taxable income as federal AGI of $106,573 minus a $9,500 standard deduction and never subtracted the $19,828.80 of taxable Social Security that Virginia exempts. It also used the standard deduction instead of $19,419.94 of itemized deductions and left out the $930 and $800 exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It made several errors. It invented a half-self-employment-tax deduction on partnership income and used a $1,000 standard deduction. It applied the $12,000 age deduction, which is fully phased out at AFAGI of $86,744. It subtracted $23,328 of Social Security even though its AGI already excluded it, a double subtraction. Finally, it submitted $2,453.71, which does not match its own computed $2,542.44."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at $80,164, when VA AGI is $86,743.77, and used a $9,000 standard deduction instead of $19,419.94 of itemized deductions. It subtracted a $12,000 age deduction that is fully phased out for a single filer with AFAGI of $86,744, and it took $745.50 of Virginia nonrefundable credits this filer does not qualify for."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model reached about $3,555 using only the $930 personal exemption, so it missed the $800 age-65 exemption. It also itemized $19,518, which skips the 0.5%-of-AGI charitable floor and counts over-the-counter expenses instead of the Medicare Part B premium. It then cut the result to $2,597 with no stated basis."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model phased out the age deduction from a $75,000 threshold, which is the married figure. For a single filer the threshold is $50,000, so the deduction is fully phased out and the model wrongly kept $256. Its itemized total of $18,985 applied the charitable floor but left out the $2,434.80 Medicare Part B premium from medical expenses."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model left the $19,829 of taxable Social Security out of federal AGI and subtracted charitable gifts above the line. It also miscalculated federal AGI as $27,752 and applied a federal standard deduction. Virginia taxes from the first dollar at 2%, so a positive taxable income cannot produce $0. The correct VA taxable income is $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed the age deduction and a retirement income subtraction would bring taxable income to zero. Virginia has no general pension exclusion, and the $12,000 age deduction is fully phased out because adjusted federal AGI of $86,744 is far above $50,000. That leaves $65,594 taxed at up to 5.75%."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model used an $8,750 standard deduction and no exemptions to get $4,227, then submitted an unrelated $1,854. It should have itemized $19,419.94 (charitable after the floor, medical including the Part B premium, and real estate tax) and deducted $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"The model started from an $8,500 standard deduction and $930 exemption, then made a vague, unquantified itemizing adjustment to get $4,014. It never computed the actual $19,419.94 of itemized deductions and left out the $800 age-65 exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model took the $8,500 VA standard deduction even though itemized deductions of $19,419.94 are larger. It also treated the personal exemptions as a credit against tax rather than a $1,730 reduction in taxable income."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model deducted only the $930 personal exemption and missed the $800 exemption for filers aged 65 or older. Its $19,518 itemized total also skips the 0.5%-of-AGI charitable floor and counts over-the-counter expenses as medical in place of the Medicare Part B premium. Together these give taxable income of $66,296 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction instead of itemizing $19,419.94. It then subtracted the $930 and $800 exemptions directly from tax as credits, when they are deductions from taxable income worth only about 5.75% of their face value."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model left taxable Social Security out of federal AGI and then claimed the full $12,000 age deduction as if income were under the limit. Adjusted federal AGI of $86,744 exceeds the $50,000 single threshold by more than $12,000, so the deduction is zero. It also omitted the $1,730 of exemptions and lowered its own $2,948 figure to $2,450 arbitrarily."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518, which leaves the $2,434.80 Medicare Part B premium out of medical expenses and skips the charitable floor. It also deducted only the $930 exemption and missed the $800 age-65 exemption, so taxable income came out at $68,296 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"The model misstated federal AGI as $104,874 and used a made-up $4,800 standard deduction instead of itemizing $19,419.94. That left taxable income at about $78,515 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted the full $12,000 age deduction, but adjusted federal AGI of $86,744 phases it out completely above the $50,000 single threshold. It also took the $8,500 standard deduction instead of itemizing $19,419.94 and omitted the $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $19,518.04 using the full $11,447 of gifts, skipping the 0.5%-of-AGI charitable floor. It also counted the $2,000 of over-the-counter expenses as medical instead of the $2,434.80 Medicare Part B premium. The two errors largely cancel but still overstate deductions by about $98, giving taxable income of $65,496 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model started from federal AGI of $106,573 without subtracting the $19,828.80 of taxable Social Security, which Virginia fully exempts. It also left the Part B premium out of medical expenses and omitted the $800 age-65 exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed $23,948.60 of itemized deductions, about $4,500 more than the allowable $19,419.94. It also deducted only the $800 age exemption and left out the $930 personal exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,180 matches taxable income of about $77,200, which is VA AGI of $86,744 minus roughly $9,550 of standard deduction and exemptions. The model did not itemize the $19,419.94 that the filer's charitable, medical (including Part B), and real estate tax amounts support."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,726 matches taxable income of about $69,280, meaning it deducted only about $17,466 from VA AGI. The full amount is $19,419.94 of itemized deductions plus $1,730 of exemptions, so the model left the Part B premium out of medical expenses and dropped most of the exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized only $15,511 (charitable plus real estate tax) and left out the $4,441.86 medical deduction for expenses above 7.5% of AGI, including the Part B premium. It also used a $1,600 exemption instead of $930 personal plus $800 age-65, which totals $1,730."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $5,617 matches taxable income of about $102,000, which is close to the full federal AGI. The model did not subtract the $19,828.80 of taxable Social Security, the $19,419.94 of itemized deductions, or the $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model figured medical as $10,000 minus 7.5% of AGI, about $2,007, and left out the $2,434.80 Medicare Part B premium that a Medicare enrollee aged 69 pays and can deduct. It also skipped the charitable floor, so itemized deductions came to $17,518 instead of $19,419.94."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,623.52 matches itemized deductions of $17,518. That figure leaves the $2,434.80 Medicare Part B premium out of medical expenses and applies no 0.5%-of-AGI charitable floor, so deductions are about $1,902 too low."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction, which allows only about $2,007 instead of $4,441.86. It also took no charitable floor, so taxable income came out at $67,496 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"The model laid out the right structure but used a standard deduction instead of itemizing. Its $14,036.62 is impossible under its own bracket schedule, which caps tax on about $77,000 of income near $4,200, so the bracket arithmetic failed outright."
+us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model miscalculated VA AGI as $87,744 instead of $86,744 and took an $8,500 standard deduction instead of itemizing $19,419.94. It used $930 for the age exemption instead of $800 and applied a made-up '$270 on the first $8,000' bracket instead of the $720 owed on the first $17,000."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed deductions and exemptions shelter all income. After the Social Security subtraction, $19,419.94 of itemized deductions, and $1,730 of exemptions, $65,593.83 of Virginia taxable income remains and is taxed up to 5.75%."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the retirement income as excluded and the filer as low-income. Virginia taxes the $39,600 of IRA and pension income in full, and the age deduction is fully phased out above $50,000 of adjusted federal AGI, which leaves $65,594 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction. It also took no 0.5%-of-AGI charitable floor, so taxable income came out at $67,496 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the Virginia age deduction, but it is fully phased out: adjusted federal AGI of $86,744 exceeds the $50,000 single threshold by more than the $12,000 deduction. That error pushed taxable income down to about $51,647 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model took the $8,750 standard deduction and only the $930 exemption. It should have itemized $19,419.94 and deducted $1,730 of exemptions, including the $800 age-65 exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,658 matches taxable income of about $85,500, which is close to the full VA AGI of $86,744. The model left out the $19,419.94 of itemized deductions and the $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the 0.5%-of-AGI charitable floor correctly but figured medical without the $2,434.80 Medicare Part B premium. As a result its itemized deductions were $16,985 instead of $19,419.94."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 matches taxable income of about $63,560, roughly $2,000 below the correct $65,594. That gap fits counting the $2,000 of over-the-counter health expenses as deductible medical expenses, which the medical deduction excludes."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model stated about $57,229 of taxable income, implying roughly $29,500 of deductions and exemptions against VA AGI. The allowable total is $21,150 ($19,419.94 itemized plus $1,730 of exemptions), so it overstated deductions by more than $8,000."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the charitable floor but figured the medical deduction without the $2,434.80 Medicare Part B premium. That gives $16,985 of itemized deductions instead of $19,419.94."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed the standard deduction and exemptions offset all tax. Virginia's deductions total $21,150 against $86,744 of VA AGI, which leaves $65,594 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction. It also skipped the 0.5%-of-AGI charitable floor, so deductions were about $1,902 too low."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $19,518 without the 0.5%-of-AGI charitable floor and counted the $2,000 of over-the-counter expenses as medical instead of the $2,434.80 Medicare Part B premium. Deductions were therefore about $98 too high, and taxable income came out at $65,496 instead of $65,594."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed the filer took the federal standard deduction and so used the $8,500 VA standard deduction. Federal itemized deductions, with real estate and state taxes as SALT plus charitable and medical, exceed the $18,150 federal standard deduction for a single filer aged 65 or older. The filer therefore itemizes in Virginia too, with $19,419.94."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted the full $12,000 age deduction, which is fully phased out above $50,000 of adjusted federal AGI. It also left out the $1,730 of exemptions and taxed income over $17,000 at 5.3% instead of Virginia's 5.75% top rate."
+us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"The model deducted only the $930 personal exemption and missed the $800 age-65 exemption. It also mis-subtracted: $86,744 minus $19,647 minus $930 is $66,167, not the $64,446 it stated."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_income_tax_before_refundable_credits, so the output was missing."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed federal non-itemizing and took the $8,500 VA standard deduction instead of itemizing $19,419.94. It also used $930 for the age-65 exemption instead of $800."
+us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the filer as a low-income retiree with no tax. VA AGI is $86,744, and after $19,419.94 of itemized deductions and $1,730 of exemptions, $65,594 remains taxable at rates up to 5.75%."
+us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model correctly treated the age deduction as zero but never subtracted the $19,828.80 of taxable Social Security from federal AGI. It also used a $9,500 standard deduction instead of itemizing $19,419.94 and deducting $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model left Social Security out of AGI and then subtracted the full $23,328 again, excluding it twice. It also invented an SE-tax deduction on partnership income, used a $1,000 standard deduction, and applied the $12,000 age deduction even though it is fully phased out at $86,744 of adjusted federal AGI."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model used a wrong federal AGI of $80,164 and applied the $12,000 age deduction, which is fully phased out. It also took a $9,000 standard deduction instead of itemizing $19,419.94 and subtracted $745.50 of nonrefundable credits that do not apply."
us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility."
us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through."
@@ -8967,55 +9189,57 @@ us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model
us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility."
us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility."
us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the $4,734 of federally tax-exempt interest as excluded from Alabama income. Alabama taxes that interest unless it comes from Alabama or U.S. obligations, so Alabama AGI is $4,734. After the $3,000 standard deduction and $1,500 personal exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable in Alabama. Alabama does not adopt the federal municipal-interest exclusion, so the $4,734 is Alabama AGI. It exceeds the $4,500 standard deduction plus personal exemption by $234, which is taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest from Alabama income and pointed to unspecified 'elderly exemptions' for age 77. Alabama includes the $4,734 of interest in AGI and has no age-based extra deduction for interest income. The $3,000 standard deduction plus $1,500 exemption leaves $234 taxable at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that tax-exempt interest is not taxed by Alabama. Alabama adds the $4,734 of federally exempt interest to Alabama AGI. Subtracting the $4,500 of standard deduction and exemption leaves $234, and 2% of that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model set Alabama AGI to $0 by grouping the tax-exempt interest with exempt Social Security. Alabama taxes that interest, so AGI is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234 of taxable income, which produces $4.68 at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said Alabama 'taxes no tax-exempt interest'. That is wrong: Alabama includes federally exempt interest from non-Alabama obligations in AGI. The $4,734 minus $4,500 of deductions leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model handled only the Social Security exemption and never counted the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated that tax-exempt interest is not included in Alabama gross income. Alabama includes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not subject to Alabama income tax, but Alabama does not follow the federal muni-interest exclusion. The $3,000 standard deduction and $1,500 exemption leave $234 taxable, which yields $4.68 at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model started from a federal AGI of $0 and never added back the $4,734 of federally tax-exempt interest, which Alabama requires. Alabama AGI of $4,734 minus $4,500 of deductions is $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and relied on nonexistent 'elderly' deductions to reach $0. Alabama includes the $4,734 in AGI. A single filer gets only the $3,000 standard deduction and $1,500 exemption, which leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model asserted that tax-exempt interest is not taxed in Alabama. Alabama taxes federally exempt interest from non-Alabama obligations, so AGI is $4,734. Subtracting $4,500 of deductions leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model correctly added back the interest but used a $4,500 'age 65+' standard deduction. Alabama has no age-65 add-on, and the single standard deduction is $3,000. With the $1,500 exemption, total deductions are $4,500, not $6,000, which leaves $234 taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed Alabama excludes federally tax-exempt interest. Alabama includes it, so AGI is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234 of taxable income, not $0, and tax at 2% is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted the $4,734 but claimed that a single filer over 65 gets at least $5,000 of standard deduction plus exemption. Alabama's single standard deduction is $3,000 and the exemption is $1,500, with no age add-on. That totals $4,500 and leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and asserted that taxable income was below a threshold. Alabama includes the $4,734 of tax-exempt interest in AGI, and the 2% bracket applies from the first dollar of taxable income. The $234 left after $4,500 of deductions produces $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model addressed only the Social Security exemption and ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stated that tax-exempt interest is excluded in Alabama. Alabama taxes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model claimed income fell below an Alabama tax threshold. Alabama includes the $4,734 of tax-exempt interest in AGI, and its 2% rate applies to the first $500 of taxable income. The $234 left after the $3,000 standard deduction and $1,500 exemption is taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model assumed that exempting Social Security leaves zero Alabama income and never counted the $4,734 of tax-exempt interest, which Alabama taxes. After $4,500 of deductions, $234 remains, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said there was no other taxable income after Social Security, overlooking that Alabama includes federally tax-exempt interest in AGI. The $4,734 minus the $3,000 standard deduction and $1,500 exemption is $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model exempted Social Security and treated that as the whole calculation, leaving out the $4,734 of tax-exempt interest that Alabama includes in AGI. Taxable income is $234 after $4,500 of deductions, which yields $4.68 at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model asserted that Alabama exempts federally tax-exempt interest. Alabama exempts only interest on Alabama and U.S. obligations, so AGI is $4,734, not $0. The $4,500 of deductions leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model kept the tax-exempt interest out of Alabama AGI. Alabama adds it back, so AGI before deductions is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model treated wages and similar items as the only Alabama-taxable income and dismissed the tax-exempt interest. Alabama includes the $4,734 in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model called taxable income 'effectively 0' without including the $4,734 of tax-exempt interest, which Alabama taxes. Alabama AGI of $4,734 minus the $3,000 standard deduction and $1,500 exemption is $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model found no taxable Alabama income after exempting Social Security and never counted the $4,734 of federally tax-exempt interest, which Alabama includes in AGI. The $4,500 of deductions leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model based Alabama taxable income on the household's zero federal taxable income. Alabama adds back the $4,734 of federally tax-exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded that the tax-exempt interest produces no Alabama taxable income. Alabama taxes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734 and taxable income after $4,500 of deductions is $234. Tax at 2% is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model excluded Social Security but found no other state-taxable income, missing that Alabama includes the $4,734 of federally tax-exempt interest in AGI. The $234 left after $4,500 of deductions is taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no other taxable income was reported, treating the $4,734 of tax-exempt interest as outside Alabama AGI. Alabama includes it, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model asserted that the tax-exempt interest does not produce Alabama taxable income. Alabama does not follow the federal exclusion, so AGI is $4,734. The $4,500 of deductions leaves $234, which yields $4.68 at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model concluded that the tax-exempt interest produces no Alabama taxable income. Alabama includes federally exempt interest from non-Alabama obligations in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model exempted Social Security and declared that there was no other taxable income, ignoring the $4,734 of tax-exempt interest that Alabama includes in AGI. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model allowed for adding back the interest but claimed the standard deduction and exemption for a single filer age 65+ fully offset it. Alabama has no age-65 add-on: the single standard deduction is $3,000 and the exemption is $1,500, so total deductions are $4,500. That leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model began from a federal AGI of $0 and never added back the $4,734 of federally tax-exempt interest, which Alabama requires. Alabama AGI of $4,734 minus $4,500 of deductions leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model required an identified out-of-state bond amount before including any interest, which reverses the default. Alabama includes all federally tax-exempt interest in AGI unless it is shown to come from Alabama or U.S. obligations, and the facts do not show that. The $4,734 minus $4,500 of deductions is $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model asserted that exemptions and deductions would cover the tax-exempt interest if it were included. Alabama does include it, and the single filer's $3,000 standard deduction plus $1,500 exemption total only $4,500 against $4,734 of AGI. The remaining $234 is taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model recognized the add-back but claimed the standard deduction and personal exemption reduce taxable income to $0. For a single filer they total only $4,500: a $3,000 standard deduction and a $1,500 exemption. Against $4,734 of AGI, that leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not taxable under Alabama law. Alabama includes federally exempt interest other than Alabama and U.S. obligations in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the interest but claimed the low-income single standard deduction plus the $1,500 exemption total at least $5,500. The single standard deduction is capped at $3,000, so the total is $4,500. Against $4,734 of AGI, that leaves $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model stated that Alabama has no state income tax. In fact Alabama levies a graduated income tax starting at 2%. The $4,734 of tax-exempt interest counts toward Alabama AGI, and the $234 left after the $3,000 standard deduction and $1,500 exemption produces $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model claimed that Alabama exempts federally tax-exempt interest. Alabama exempts only interest on Alabama and U.S. obligations, so AGI is $4,734, not $0. After $4,500 of deductions, $234 is taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model labeled the $4,734 of tax-exempt interest as not taxable in Alabama. Alabama includes federally exempt interest in AGI. The $3,000 standard deduction and $1,500 exemption leave $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model concluded that taxable income is $0 after the standard deduction without counting the $4,734 of federally tax-exempt interest, which Alabama includes in AGI. The $3,000 standard deduction and $1,500 exemption total only $4,500, leaving $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the $4,734 of federally tax-exempt interest as also exempt in Alabama and got Alabama AGI of $0. Alabama adds this interest back into gross income, so AGI is $4,734. After the $3,000 standard deduction and $1,500 personal exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable in Alabama, but Alabama includes the $4,734 of federally exempt interest in Alabama AGI. Only $4,500 of deductions offset it ($3,000 standard deduction plus $1,500 exemption), so $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest from Alabama income and assumed 'elderly exemptions' wipe out the liability. Alabama taxes the $4,734 of federally exempt interest and gives no extra age-based deduction. Deductions total only $3,000 plus $1,500, leaving $234 taxable and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and assumed income falls below the deduction and exemption threshold. Alabama includes the $4,734 of interest in AGI, and the $3,000 standard deduction plus $1,500 personal exemption total only $4,500, with no age-77 increase. That leaves $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as untaxed in Alabama. Alabama adds federally exempt interest back into its gross income. After the $4,500 of standard deduction and personal exemption, $234 remains and is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model got Alabama AGI of $0 by excluding both Social Security and the tax-exempt interest. Only Social Security is exempt; the $4,734 of federally exempt interest is Alabama income. Subtracting $3,000 plus $1,500 leaves $234 taxable, for $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model claimed Alabama 'taxes no tax-exempt interest'. Alabama includes the $4,734 of federally exempt interest in AGI, and after the $4,500 standard deduction and personal exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and ignored the $4,734 of tax-exempt interest. Alabama adds that interest to AGI. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234 taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said tax-exempt interest is not included in Alabama gross income, but Alabama does include federally exempt interest. That makes AGI $4,734, minus $4,500 of standard deduction and personal exemption, leaving $234 taxable and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as outside Alabama's tax base. Alabama taxes the $4,734 of federally exempt interest, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model carried federal AGI of $0 straight into Alabama. Alabama adds back the $4,734 of federally tax-exempt interest, so Alabama AGI is $4,734. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and assumed 'elderly deductions' exceed any income. Alabama includes the $4,734 of interest in AGI and has no age-based standard deduction add-on. The $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxed in Alabama, but Alabama adds federally exempt interest to gross income. That makes AGI $4,734, minus the $4,500 standard deduction and personal exemption, leaving $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model correctly allowed that the interest might count as Alabama income, then used a nonexistent $4,500 age-65 standard deduction. Alabama's single standard deduction is $3,000 with no elderly add-on, so total deductions are $4,500, not $6,000. That leaves $234 taxable and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said Alabama excludes federally tax-exempt interest, but Alabama includes it in gross income. That makes AGI $4,734, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted the $4,734 as income but claimed the standard deduction plus exemption is 'at least $5,000' for a single filer over 65. Alabama gives no age-based increase: the deductions are $3,000 plus $1,500, or $4,500. That leaves $234 taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model assumed taxable income falls below the liability threshold without counting the $4,734 of tax-exempt interest, which Alabama includes in AGI. After only $4,500 of standard deduction and personal exemption, $234 is taxable at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and left out the $4,734 of federally tax-exempt interest, which Alabama taxes. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model treated tax-exempt interest as excluded in Alabama. Alabama adds the $4,734 of federally exempt interest to AGI, leaving $234 after $4,500 of deductions, which gives $4.68 at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model asserted income falls below Alabama's threshold without including the $4,734 of tax-exempt interest, which Alabama counts in AGI. Alabama's $3,000 standard deduction and $1,500 exemption cover only $4,500, leaving $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped after exempting Social Security and ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the head has no other taxable income, but Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxable, which gives $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model exempted Social Security but never added the $4,734 of tax-exempt interest to Alabama AGI as Alabama law requires. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model claimed Alabama exempts federally tax-exempt interest, but Alabama includes it in gross income. That makes AGI $4,734 rather than $0, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said tax-exempt interest is not in the Alabama AGI base, but Alabama adds the $4,734 of federally exempt interest back in. After the $4,500 standard deduction and personal exemption, $234 is taxable at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as non-taxable in Alabama. Alabama includes the $4,734 of federally exempt interest in AGI, which leaves $234 taxable after $4,500 of deductions and gives $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed taxable income is effectively zero without including the $4,734 of tax-exempt interest, which Alabama taxes. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model listed no taxable Alabama income, overlooking that Alabama includes the $4,734 of federally tax-exempt interest in AGI. The $3,000 standard deduction and $1,500 exemption cover only $4,500, leaving $234 taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model used zero federal taxable income as the Alabama base. Alabama adds back the $4,734 of federally exempt interest, giving AGI of $4,734. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said the tax-exempt interest produces no Alabama taxable income, but Alabama includes all $4,734 in AGI. The $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model concluded the listed facts establish no Alabama-taxable income. The $4,734 of federally tax-exempt interest is Alabama income, and after $4,500 of standard deduction and exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no other taxable income was reported, ignoring that Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said the tax-exempt interest yields no Alabama taxable income, but Alabama includes the $4,734 in AGI. Deductions of $4,500 leave $234 taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as producing no Alabama taxable income. Alabama adds the $4,734 of federally exempt interest to AGI, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxable, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said there is no taxable income besides Social Security, overlooking that Alabama taxes the $4,734 of federally exempt interest. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model added back the interest but assumed the age-65+ standard deduction and exemptions fully offset $4,734. Alabama's single standard deduction is $3,000 with no age add-on, plus a $1,500 personal exemption, for $4,500 total. That leaves $234 taxable and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model carried federal AGI of $0 into Alabama, but Alabama adds back the $4,734 of federally exempt interest. The $3,000 standard deduction and $1,500 exemption leave $234 taxable at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model excluded the interest because no out-of-state bond source was identified. Under the modeled rule, all $4,734 of federally tax-exempt interest is Alabama income. After the $4,500 standard deduction and exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model assumed exemptions and deductions would be enough to absorb the interest if it were included. Alabama's deductions total only $4,500 ($3,000 standard plus $1,500 personal exemption), which is less than $4,734. That leaves $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added back the tax-exempt interest but claimed the standard deduction and personal exemption reduce taxable income to $0. They total only $4,500 ($3,000 plus $1,500), so $234 remains taxable at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable under Alabama law, but Alabama includes the $4,734 of federally exempt interest in AGI. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model added back the interest but used a standard deduction of at least $4,000, for $5,500 total. Alabama's single standard deduction is $3,000 at this income level, so deductions total $4,500. That leaves $234 taxable and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Alabama has no state income tax, but Alabama levies a graduated income tax starting at 2%. Its AGI includes the $4,734 of federally exempt interest, and after the $3,000 standard deduction and $1,500 exemption, the tax is $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model said Alabama exempts federally tax-exempt interest, but Alabama includes it in AGI, making AGI $4,734 rather than $0. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not taxable in Alabama, but Alabama includes it in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said taxable income after the standard deduction is $0, either leaving out the $4,734 of interest Alabama counts in AGI or overstating the deductions. Alabama's $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68."
us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount."
-us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented expanded or special disabled-adult coverage and treated low assets and potentially favorable treatment of veterans benefits as sufficient. Florida assigns the spouse no Medicaid category, so neither the income comparison nor the $4,870 resource amount creates eligibility."
-us,scenario_116,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the disability flag as an automatic Medicaid eligibility pathway. The spouse must first qualify under a Florida Medicaid category, and the engine assigns medicaid_category = NONE."
+us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model itself said the $52,800 in veterans benefits exceeds Florida's disabled-adult income limit, then answered eligible anyway because Florida 'typically provides expanded Medicaid' and has unnamed 'special provisions'. Florida is a non-expansion state with no such override. The spouse gets no SSI and fails the aged/disabled income test, so no category applies."
+us,scenario_116,spouse_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model assumed veterans benefits are excluded from income for the disability pathway. That exclusion applies only to MAGI, and in non-expansion Florida MAGI gives a childless adult no category. The SSI-related disabled pathway counts VA benefits as unearned income, so the $52,800 cuts SSI to $0 and fails Florida's aged/disabled income limit."
+us,scenario_116,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated being disabled as enough on its own to qualify for Medicaid and applied no income test. Florida's disabled pathway requires SSI receipt or income under the aged/disabled limit. The spouse's $52,800 in veterans benefits fails both tests, and Florida offers no expansion adult category as a fallback."
us,scenario_116,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated that under-65 Medicare requires 24 months of SSDI entitlement. It then assumed the spouse receives SSDI and has met the waiting period, but the household lists no Social Security disability income, so SSDI is 0 and the SSDI-months test fails. It also treated the $52,800 in VA veterans benefits as disability benefits for Medicare purposes, but VA payments are not Title II SSDI and do not start the 24-month clock."
us,scenario_116,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied disability-based Medicare eligibility from the 'is disabled' flag alone. PolicyEngine's rule requires age 65+ or at least 24 months of SSDI receipt. The 42-year-old spouse has no SSDI income or SSDI months (both 0), so the spouse is not Medicare eligible."
us,scenario_116,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied disability-based Medicare eligibility because a disability was listed. It skipped PolicyEngine's actual non-age pathway, which is at least 24 months of Social Security disability receipt. The spouse is 42 and has zero SSDI, since only VA veterans benefits are listed, so eligibility is No."
@@ -9027,250 +9251,257 @@ us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categori
us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold."
us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies."
us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Phased the qualified-overtime deduction out from the $150,000 single-filer threshold, which cut it to $15,185. Joint filers phase out only above $300,000 MAGI, so the full $25,000 cap applies at $250,733. It also dropped the taxable $2,589 state refund and skipped the $2,000 non-itemizer charitable deduction. It then subtracted a $1,200 CDCC even though the head's zero earned income sets the §21(d) expense limit to $0."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Phased the overtime deduction out from $150,000, the single-filer threshold, which gave $14,927 instead of the full $25,000 allowed to joint filers below $300,000 MAGI. It also chose to itemize $33,088 by comparing only against the bare $32,200 standard deduction. It missed the $2,000 non-itemizer charitable deduction, which makes the standard path worth $34,200. Taxable income came out at $202,719 instead of $191,533."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Added the $35,525 overtime premium on top of $248,675 of wages that already include it. It subtracted the $25,900 charitable gifts and $4,752 of employee business expenses above the line, used a $29,200 standard deduction, and never applied the $25,000 qualified-overtime deduction. Its $26,809 is the pre-credit tax on an inflated $226,407 of taxable income, with none of the $6,600 CTC subtracted."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Invented about $35,070 of mortgage interest from the $501,000 balance; no interest paid was listed, so the deduction is $0. It combined that with a $10,000 SALT figure to itemize and omitted the $25,000 qualified-overtime deduction. It also gave the 16-year-old a $500 other-dependent credit and used $2,000 per child instead of $2,200 for all three under-17 children. It claimed a $600 CDCC despite the head's zero earnings, and its near miss comes from these errors offsetting each other."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Submitted $47,714, which contradicts its own derivation ending near $25,972. That derivation also made up $20,000 of mortgage interest from the balance, omitted the $25,000 qualified-overtime deduction, and used $2,000 rather than $2,200 per child. It claimed a CDCC even though the head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Built an itemized deduction of about $80,000 from imputed mortgage interest (about $32,000, though no interest was listed) and about $40,000 of SALT. It then subtracted the entire $35,525 overtime premium instead of the $25,000 joint cap, pushing taxable income down to about $135,000 instead of $191,533. It also used $2,000 per child and claimed a $1,200 CDCC that the zero-earning head blocks."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Chose to itemize $33,186 (Arkansas income tax plus floored charity) because it beat the bare $32,200 standard deduction. It missed the 2026 non-itemizer charitable deduction of $2,000 for joint filers, which makes the standard path worth $34,200. The $1,014 of lost deductions, taxed at 22%, accounts exactly for its $223 overstatement."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $25,000 qualified-overtime deduction entirely, used the outdated $30,000 standard deduction and 2025 brackets, and took no $2,000 non-itemizer charitable deduction, so it taxed $220,734 instead of $191,533. It also used $2,000 per child instead of $2,200. It subtracted a $1,200 CDCC even though the head's zero earned income caps creditable expenses at $0."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Submitted $62,700, which contradicts its own worked result of $22,630. That worked result also imputed about $32,000 of mortgage interest from a balance with no listed interest, omitted the $25,000 qualified-overtime deduction, and used $2,000 per child. It claimed a $1,200 CDCC that the head's zero earned income rules out."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Limited the qualified-overtime deduction to $12,500, the single-filer cap. A joint return gets the $25,000 cap no matter which spouse earned the overtime, and MAGI is below the $300,000 phaseout. It also itemized about $35,400 (including a medical deduction) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, so taxable income was $202,834 instead of $191,533."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Imputed $32,565 of mortgage interest from the bare balance and omitted the $25,000 qualified-overtime deduction. It applied only $900 of nonrefundable CTC, although the pre-credit tax absorbs the full $6,600 ($2,200 × 3); a refundable portion exists only when the credit exceeds liability. It also subtracted a $1,200 CDCC that the head's zero earned income disallows."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a pre-TCJA 15%/25%/28% rate schedule and treated the CTC as fully phased out. It ignored the permanent post-2025 regime: 10/12/22% brackets, a $2,200-per-child CTC that phases out only above $400,000 joint MAGI, and the $25,000 qualified-overtime deduction. As a result it taxed $214,093 and subtracted no credit, when the correct figures are $191,533 of taxable income and a $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Excluded the entire $35,525 FLSA overtime premium from AGI instead of taking the below-the-line qualified-overtime deduction capped at $25,000 for joint filers, overstating the deduction by $10,525. It then itemized $36,871, including a $3,905 medical deduction, instead of taking the standard deduction plus the $2,000 non-itemizer charitable deduction. It also used $2,000 per child instead of $2,200."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, using only a $31,500 standard deduction. It counted just $600 of the CTC as nonrefundable. The nonrefundable CTC is the full credit up to tax liability, which here is the entire $6,600 (the model used $6,000), with nothing carved out as refundable."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: five $5,000 personal exemptions, pre-TCJA brackets, and a $1,000-per-child CTC. Post-2025 law keeps the $32,200 standard deduction, 10/12/22% brackets, and a $2,200 CTC, with no exemptions. It also omitted the $25,000 qualified-overtime deduction and claimed a $1,200 CDCC that the zero-earning head blocks."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. The correct path takes the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions to reach $191,533 of taxable income and $31,561 of tax, less the $6,600 CTC. Its $46,522 exceeds even the $37,164 pre-credit tax with the overtime deduction dropped, so it omitted the overtime deduction and never subtracted the CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA 2026 rules instead of the permanent post-2025 regime, so it missed the $2,200-per-child CTC, the $25,000 qualified-overtime deduction, and the $2,000 non-itemizer charitable deduction. It also applied a CDCC, although the head's zero earned income sets the creditable-expense limit to $0."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Used TCJA-expiration rules with personal exemptions and a $1,000 CTC phased out above the old $110,000 threshold. 2026 law gives $2,200 per child and phases out only above $400,000 joint MAGI, so the full $6,600 applies. It also omitted the $25,000 qualified-overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. Its $35,283 is higher than the $30,564 obtained even with the $25,000 qualified-overtime deduction dropped. It therefore omitted the overtime deduction and did not take the full $6,600 CTC off the $31,561 tax on $191,533 of taxable income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Invoked a CTC phaseout that does not apply, since joint MAGI of $250,733 is far below the $400,000 threshold. The full $6,600 CTC comes off the $31,561 bracket tax on $191,533 of taxable income. Its $26,521.84 keeps about $1,560 of liability that this nonexistent phaseout left uncredited."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. Its $41,320 exceeds the $37,164 pre-credit tax computed with the overtime deduction dropped, so it omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction and subtracted none of the $6,600 CTC. The correct figure is $31,561 of tax on $191,533 of taxable income, less the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Started from $248,145 AGI, leaving out the taxable $2,589 state refund. Its $33,816 is higher than the $30,564 produced even without the overtime deduction, so it omitted the $25,000 qualified-overtime deduction and did not subtract the full $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction for the spouse's $35,525 FLSA premium and the $2,000 non-itemizer charitable deduction, so it taxed $219,734 instead of $191,533. It also used $2,000 per child instead of the 2026 CTC of $2,200."
-us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction, giving taxable income of $215,834 instead of $191,533. It treated only $500 per child as nonrefundable CTC, although the full $2,200 per child offsets liability and the refundable ACTC applies only to credit that exceeds tax. It also subtracted a $1,200 CDCC that the head's zero earned income disallows."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Its $32,981 is higher than the $30,564 obtained even with no overtime deduction. It therefore never applied the $25,000 qualified-overtime deduction (joint cap, no phaseout below $300,000) or the $2,000 non-itemizer charitable deduction. It also counted a childcare credit that the head's zero earned income disallows."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Reported zero liability, but nothing eliminates the tax. After the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions, taxable income is $191,533 and tax is $31,561. The only nonrefundable credit is the $6,600 CTC, which leaves $24,961."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction for the spouse's $35,525 FLSA premium (joint cap, MAGI below the $300,000 phaseout), so taxable income stayed at $214,996 instead of $191,533. It also itemized $35,738 instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its value reconstructs exactly: $250,734 − $32,200 standard − $12,500 overtime = $206,034 taxable, then $34,751 of tax less two $2,200 CTCs and a $1,200 CDCC. It applied the single-filer $12,500 overtime cap instead of the $25,000 joint cap and skipped the $2,000 non-itemizer charitable deduction. It also dropped the 16-year-old, who is under 17 and qualifies for the CTC, and claimed a CDCC that the zero-earning head blocks."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Got taxable income ($191,534), bracket tax ($31,561), and the $6,600 CTC right, then subtracted a $600 CDCC. The CDCC is $0 because IRC §21(d) caps creditable expenses at the lower-earning spouse's earned income, and the head has none."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $33,895 (charity plus SALT) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200). That left taxable income about $305 too high and the tax $67 over."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $33,080 (floored charity plus Arkansas income tax) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction worth $34,200. That left taxable income about $1,121 too high and the tax $247 over."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its answer implies itemized deductions of about $35,434, which is $1,234 more than the household can itemize once charity is reduced by the 0.5%-of-AGI floor and medical costs fall under the 7.5% floor. The household takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction instead, so taxable income is $191,533, not about $190,300."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used the $32,200 standard deduction and the full $25,000 overtime deduction but omitted the 2026 non-itemizer charitable deduction of $2,000 for joint filers. That left taxable income at $193,534 and the tax $440 too high."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $34,340 (floored charity plus an Arkansas income-tax estimate) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. Its state-tax figure overstates what can be itemized and pushes the total past $34,200, cutting taxable income $139 below the correct $191,533."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. $52,000 is higher than the $37,164 pre-credit tax even with the overtime deduction dropped, and even above the roughly $45,400 that 2026 brackets would levy on the full $250,733 AGI with no deductions. It therefore omitted the $25,000 qualified-overtime deduction, the $2,000 charitable deduction, and the $6,600 CTC, and used rates above the 2026 schedule."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (personal exemptions, a 25% bracket, and a fully phased-out CTC) instead of post-2025 law. Current law keeps 10/12/22% brackets and a $2,200 CTC that phases out only above $400,000 joint MAGI. It also omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly applied TCJA-sunset law: five $5,300 exemptions, uncapped SALT, 15%/25% brackets, and a CTC phased out above $110,000. 2026 law keeps the $32,200 standard deduction, 10/12/22% brackets, and a $2,200 CTC with a $400,000 joint threshold, so the full $6,600 applies. It also omitted the $25,000 qualified-overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Reached the correct $191,534 of taxable income and the $6,600 CTC but computed bracket tax of $31,735 instead of $31,561. The correct sum is 10% of $24,800 ($2,480) + 12% of $76,000 ($9,120) + 22% of $90,734 ($19,961). The $174 arithmetic error carries straight into the answer."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted the $8,223 ESI premium from AGI, applied TCJA-sunset personal exemptions and 15%/25% brackets, zeroed the CTC as phased out, and omitted the $25,000 qualified-overtime deduction. Under 2026 law the full $6,600 CTC applies below the $400,000 joint threshold. It also claimed a $1,200 CDCC that the head's zero earned income blocks."
-us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Inflated itemized deductions to $37,873. It took the full $25,900 of charity without the 2026 0.5%-of-AGI floor, added a $3,116 medical deduction although total medical costs do not clear the $18,805 (7.5%-of-AGI) floor, and added a $237 miscellaneous deduction that remains suspended. The household takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, so taxable income is $191,533, not $187,861."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $34,415, including a $741 medical deduction that counts the head's $8,223 premium twice; the 'excluding Part B' and 'other' premium lines describe the same premium. Without the double count, medical costs ($11,823) stay under the $18,805 floor, and itemizing ($33,674) loses to the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. That gives $191,533 of taxable income."
-us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed that credits offset all liability. The only nonrefundable credit is the $6,600 CTC against $31,561 of tax on $191,533 of taxable income, and no CDCC applies because the head has no earnings. That leaves $24,961."
-us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Its derivation reached about $24,980 but it submitted a rounded $25,000. The derivation also itemized the full $25,900 of charity without the 0.5%-of-AGI floor, offset by a low Arkansas-tax guess, instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. It ended up $38.66 above $24,961.34."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Imputed $24,375 of mortgage interest from the balance, though no interest paid was listed, so the deduction is $0. It also took a flat $10,000 SALT and omitted the $25,000 qualified-overtime deduction, then used $2,000 per child plus a $600 CDCC. The invented mortgage interest roughly stood in for the missing overtime deduction, and the invalid $600 CDCC replaced the missing $600 of CTC, so offsetting errors left it only $144 off."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Excluded the full $35,525 overtime premium from wages and invented rules that do not exist: a '$2,800 additional standard deduction for net overtime', a SALT-cap 'excess taxes' reduction, an 'overtime deduction credit', and a $409,000 mortgage cap. The correct computation takes the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions to reach $191,533 of taxable income and $31,561 of tax, less the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the overtime deduction starting at $150,000 AGI, which is the single-filer threshold; the joint threshold is $300,000. That cut the deduction to $15,185 instead of the full $25,000. It also left out the $2,000 non-itemizer charitable deduction and the taxable $2,589 state refund, and it subtracted a $1,200 CDCC even though Head has no earned income, so the CDCC is $0."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced the overtime deduction by 10% of MAGI above $150,000, leaving $14,927. For joint filers the phaseout starts at $300,000, so the full $25,000 is allowed. It also itemized $33,088 instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200). Together these overstated taxable income by $11,185."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of wages, even though wages already include it. It treated charitable gifts and employee business expenses as above-the-line deductions, used an outdated $29,200 standard deduction, and never took the $25,000 qualified-overtime deduction. It also never subtracted the $6,600 nonrefundable CTC, so its figure is essentially a guess at pre-credit tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It said tax is 'well above zero' and 'not computed in detail', then submitted 0 as a placeholder. The correct calculation is $191,533.36 of taxable income, $31,561.34 of tax, and $6,600 of CTC, which gives $24,961.34."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $35,070 of mortgage interest from the loan balance, although no interest paid is listed, so that input is 0. It also claimed a $10,000 SALT amount and itemized $70,970, while leaving out the $25,000 qualified-overtime deduction. On credits, it used 2025 brackets and $2,000 per-child CTC, wrongly gave the 16-year-old only a $500 ODC (a 16-year-old is a qualifying child), and claimed a $600 CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Its own reasoning reached about $25,972, but it submitted $47,714, which contradicts that work. The reasoning itself assumed mortgage interest from the balance, left out the $25,000 overtime deduction, used $2,000 per-child CTC, and claimed a CDCC that Head's zero earnings block."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed about $80,000 of itemized deductions, including about $32,000 of mortgage interest even though no interest is listed. It deducted the full $35,525 overtime premium instead of capping the deduction at $25,000. It also used $2,000 per-child CTC and a $1,200 CDCC, but the CDCC is $0 because Head has no earned income."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $33,186 because that beat the $32,200 standard deduction. It missed the 2026 $2,000 non-itemizer charitable deduction for joint filers, which raises the standard path to $34,200. The missed $1,014 of deductions at 22% accounts for its $223 overstatement."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction. It used a $30,000 standard deduction and 2025 brackets in place of the 2026 figures of $32,200 and a 22% bracket ending at $211,400. It also used $2,000 per-child CTC and claimed a $1,200 CDCC, which Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reached $22,630, but it submitted $62,700, which has no basis in its own calculation. The reasoning itself assumed $32,000 of mortgage interest that is not in the facts, left out the $25,000 overtime deduction, used $2,000 per-child CTC, and claimed a CDCC that Head's zero earnings block."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed only $12,500 of qualified-overtime deduction, which is the single-filer cap. Joint filers get up to $25,000, and this household is under the $300,000 phaseout. It also itemized about $35,400 instead of taking the $34,200 standard-plus-non-itemizer-charitable path, which left taxable income about $11,300 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $32,565 of mortgage interest from the loan balance, although no interest is listed, and left out the $25,000 qualified-overtime deduction. It applied only $900 of nonrefundable CTC instead of the full $6,600, which the large tax liability absorbs completely. It also claimed a $1,200 CDCC that Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets (15%, 25%, 28%) and treated the CTC as fully phased out, but OBBBA made the TCJA rate structure and the $2,200 CTC with a $400,000 phaseout permanent for 2026. It also left out the $25,000 overtime deduction and itemized instead of taking the $32,200 standard deduction plus the $2,000 charitable deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It removed the entire $35,525 overtime premium from AGI. The qualified-overtime deduction is a below-the-line deduction capped at $25,000. It also itemized with an overstated $3,905 medical deduction and used $2,000 instead of $2,200 per child for the CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, and it used the 2025 $31,500 standard deduction. It also treated only $600 of the CTC as nonrefundable. The full $6,600 CTC offsets regular tax first, and only the unused part goes to the refundable ACTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with $5,000 personal exemptions, pre-TCJA rates, and a reduced CTC. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 CTC permanent. It also left out the $25,000 overtime deduction and claimed a CDCC that Head's zero earned income blocks."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $46,522 is higher than the tax on the entire $250,733 AGI with no deductions at all. That means it left out the standard deduction, the $25,000 overtime deduction, and the $6,600 CTC. The correct calculation is $191,533.36 taxable income, $31,561.34 of tax, minus $6,600 of CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules, but OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 CTC permanent. It also left out the $25,000 qualified-overtime deduction and applied a CDCC that is $0 because Head has no earned income."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired, which led it to use personal exemptions, pre-TCJA brackets, and a $1,000 CTC phased out at this income. Under the permanent OBBBA rules, the $2,200-per-child CTC starts phasing out only at $400,000 for joint filers, so the full $6,600 applies. It also left out the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation, and its $35,283 is about $10,300 above the correct figure. That gap matches leaving out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction and not subtracting the full $6,600 CTC. The correct calculation is $191,533.36 taxable income and $31,561.34 of tax, minus $6,600."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a CTC phaseout, but the $400,000 joint phaseout threshold is far above this household's $250,733 AGI. The full $6,600 CTC therefore applies against $31,561.34 of tax. Its $1,560 overstatement comes from that reduced credit and from understated deductions."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,320 is higher than the tax on standard-deduction-only income, so it left out both the $25,000 qualified-overtime deduction and the $6,600 nonrefundable CTC. The correct calculation is $191,533.36 taxable income and $31,561.34 of tax, minus $6,600."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the taxable $2,589 state refund, so its AGI was $248,145. It also left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, which pushed tax about $8,900 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took only a roughly $31,000 standard deduction and left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction. That left taxable income at $219,734 instead of $191,533. It also used $2,000 per child for the CTC instead of the 2026 amount of $2,200."
+us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction. It also treated only $500 per child of the CTC as nonrefundable, but the whole $2,200-per-child credit first offsets regular tax, which is large enough to absorb all $6,600. It also subtracted a $1,200 CDCC, which Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the standard deduction and left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, together worth about $5,940 at 22%. It also cited a childcare credit, which is $0 because Head has no earned income."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed deductions and credits wipe out the liability completely. In fact, taxable income is still $191,533.36 after all deductions, giving $31,561.34 of tax, and the only nonrefundable credit is the $6,600 CTC, which leaves $24,961.34."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $214,996 shows it never took the $25,000 qualified-overtime deduction for the spouse's FLSA overtime premium. It also itemized instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. Those errors left tax $5,234 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its result reconstructs as a $12,500 overtime deduction, which is the single cap instead of the $25,000 joint cap, with no $2,000 non-itemizer charitable deduction. It allowed CTC for only two children, although the children aged 16, 11, and 2 are all under 17, and it subtracted a $1,200 CDCC that Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income and $6,600 CTC match the correct calculation. However, it subtracted a $600 CDCC. Under IRC §21(d), creditable expenses are limited to the lower-earning spouse's earned income, and Head's earned income is $0, so the CDCC is $0."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized charitable gifts and Arkansas tax totaling about $33,900. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), so its taxable income was about $305 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,080 of charity (after the 0.5% floor) and state tax. It missed that the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction gives $34,200, so taxable income came out about $1,120 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $35,430 of charitable and state-tax deductions, overstating the itemized total. The correct federal deduction path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), which gives taxable income of $191,533.36, so its tax came out $271 too low."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $32,200 standard deduction and the $25,000 overtime deduction but left out the 2026 $2,000 non-itemizer cash charitable deduction for joint filers. That left taxable income at $193,534 instead of $191,533.36 and tax $440 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $34,340 of charity and estimated Arkansas tax, overstating the state-tax amount. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), so its taxable income was about $139 too low."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave a round guess of $52,000 with no calculation. That figure is higher than the tax on the full AGI with no deductions or credits, so it ignored the $32,200 standard deduction, the $25,000 overtime deduction, the $2,000 charitable deduction, and the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-TCJA' sunset law: personal exemptions, a 25% bracket, and a CTC fully phased out. OBBBA made the TCJA brackets permanent and set a $2,200 CTC with a $400,000 joint phaseout. It also left out the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled TCJA expiration, using pre-TCJA 15%/25% brackets, $5,300 exemptions, uncapped SALT, and a $110,000 CTC phaseout. Under OBBBA's permanent rules, the 2026 brackets top out at 22% for this income and the full $6,600 CTC applies. It also left out the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $191,534 and $6,600 CTC are right. However, it computed $31,735 of tax instead of $31,561.48. The 2026 joint brackets give $2,480 + $9,120 + 22% × $90,734 = $31,561.48, so its bracket arithmetic was off by $174."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law (personal exemptions and 15%/25% brackets) and dropped the CTC entirely, and it also subtracted Head's $8,223 premium from the spouse's wages. Under permanent OBBBA law, the $25,000 overtime deduction and the $6,600 CTC apply. It also claimed a $1,200 CDCC, which Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated itemized deductions at $37,873. It took the full $25,900 of charity without the 2026 0.5%-of-AGI floor, used a $3,116 medical deduction, and included $237 of miscellaneous employee business expenses, which are not deductible. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200)."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to grade."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared $34,414.79 of itemized deductions against only the $32,200 standard deduction, leaving out the $2,000 non-itemizer charitable deduction that raises the standard path to $34,200. It also overstated the Arkansas income tax part of SALT at $9,027.51. Under the correct calculation the household takes the standard path, with taxable income of $191,533.36."
+us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits wipe out all tax, but the only nonrefundable credit is the $6,600 CTC. That covers only part of the $31,561.34 of tax on $191,533.36 of taxable income, leaving $24,961.34."
+us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Its own numbers ($31,580 of tax minus $6,600) give about $24,980, but it submitted a rounded $25,000. It also itemized the full $25,900 of charity without the 0.5% AGI floor and compared that total against the $32,200 standard deduction alone. The correct deduction is $32,200 plus the $2,000 non-itemizer charitable deduction ($34,200), giving exactly $24,961.34."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $24,375 of mortgage interest using a fictitious $375,000 debt cap (the acquisition-debt cap is $750,000, and no interest is listed anyway) and itemized $60,275. It left out the $25,000 qualified-overtime deduction, used $2,000 per-child CTC, and claimed a $600 CDCC that Head's zero earned income rules out. These offsetting errors happened to land near the correct figure."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the whole $35,525 overtime premium from wages, invented an 'additional standard deduction for net overtime' and an 'overtime deduction credit', used a 32% marginal rate, and applied a nonexistent $409,000 mortgage-debt cap. The actual rule is a below-the-line qualified-overtime deduction capped at $25,000, which leaves taxable income of $191,533.36 in the 22% bracket."
us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It never applied the rule that the ACTC can refund only the CTC left after offsetting tax liability. A federal tax bill of more than $25,000 absorbs all of this household's CTC, so the ACTC is $0. Instead it invented a $3,750 amount 'limited by the ratio of Additional CTC to total CTC'. Along the way it counted the FLSA overtime premium twice on top of gross wages, used the 2023 $27,700 standard deduction and $2,000-per-child CTC, and said income was above the $400,000 phase-out when AGI is about $250,733."
us,scenario_117,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"It treated the $1,700-per-child refundable cap as an amount paid out automatically ($5,100). It skipped the step that limits the ACTC to CTC left after tax liability. With more than $25,000 of federal income tax, the full $6,600 CTC is used as a nonrefundable credit, so the refundable portion is $0."
us,scenario_117,federal_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It multiplied 3 children by an $1,800 refundable cap to get $5,400. It never limited the ACTC to CTC left after the credit offsets tax liability. The household's federal tax of more than $25,000 absorbs the full $6,600 CTC, so the ACTC is $0. EITC is also $0."
us,scenario_117,federal_refundable_credits,glm-5.3,llm_error,other,False,"It stated the rule backwards, saying that 'with ample tax liability' the household claims the full refundable CTC of $1,700 per child ($5,100). In fact, ample tax liability is exactly what reduces the ACTC to zero: the refundable portion is limited to CTC left after offsetting tax. More than $25,000 of federal tax absorbs the entire $6,600 CTC, so refundable credits are $0."
us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_refundable_credits, so there is no substantive calculation to score. Under the correct derivation, the full $6,600 CTC offsets this household's more-than-$25,000 federal tax, which leaves a $0 ACTC. EITC is $0 as well."
us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a household-level “young-child context” pathway and transferred the two-year-old child's WIC category to the 44-year-old head. The head has no listed pregnancy, postpartum, or breastfeeding status and is neither an infant nor a child under five; independently, household income exceeds the 185% poverty limit."
-us,scenario_117,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"Its own arithmetic reached the reference — 6.2% on the $184,500 2026 Social Security taxable maximum = $11,439 plus 1.45% × $248,675 = $3,605.79, totaling $15,044.79 — and it then overrode that with an unfounded $183,600 projected wage base for a stated $14,988.99. It finally submitted $12,896.44, which no stated step produces: net of the correct $3,605.79 Medicare tax it implies a $149,849 wage base, far below any 2026 figure."
-us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It added the $35,525 FLSA overtime premium to the $248,675 of gross wages even though the prompt states gross wages already include overtime pay, used the $168,600 2024 taxable maximum instead of 2026's $184,500, and applied the 0.9% Additional Medicare Tax at the $200,000 single-filer threshold when the married-filing-jointly threshold of $250,000 governs and combined wages fall $1,325 below it. The dominant error is a fabricated $17,052 'Arkansas payroll tax' at 6.0%: Arkansas withholding is state income tax, and Arkansas imposes no mandatory employee-side payroll tax, plus an invented $1,680 'additional mandatory AR payroll withholding'."
-us,scenario_117,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"It computed the reference exactly — $184,500 × 6.2% = $11,439 plus $248,675 × 1.45% = $3,605.79 = $15,044.79 — then discarded the correct 2026 taxable maximum and walked the wage base down through successively lower guesses ($183,600, $181,800, $179,800) before submitting $14,757.34. The $287.45 shortfall is 6.2% of the $4,636 it shaved off the $184,500 taxable maximum."
-us,scenario_117,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"Its stated derivation is the reference verbatim: $11,439 of Social Security tax on the $184,500 2026 taxable maximum, $3,605.79 of Medicare tax on $248,675, no Additional Medicare Tax below the $250,000 joint threshold, and no Arkansas employee payroll tax, totaling $15,044.79. It then submitted $13,347.55 after an unstated 'recompute with SS base estimate'; that figure implies a $157,125 wage base, below the 2025 maximum, let alone 2026's."
-us,scenario_117,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"It used $176,100 — the 2025 Social Security taxable maximum — as the 2026 wage base, capping employee OASDI tax at $10,918.20 instead of the $11,439 produced by the 2026 maximum of $184,500. Every other element (uncapped 1.45% Medicare on $248,675, no Additional Medicare Tax under the $250,000 joint threshold, no Arkansas employee payroll tax) is right, so the entire $520.80 shortfall is 6.2% of the $8,400 by which it understated the wage base."
-us,scenario_117,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"It substituted an invented $178,200 projection for the 2026 Social Security taxable maximum of $184,500, capping employee OASDI tax at $11,048.40 rather than $11,439. Its Medicare tax ($3,605.79), Additional Medicare Tax treatment (none, since $248,675 is under the $250,000 joint threshold), and Arkansas result (no employee payroll tax) are correct, leaving a shortfall of exactly 6.2% × $6,300 = $390.60."
-us,scenario_117,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It applied a guessed $181,500 wage base instead of the 2026 Social Security taxable maximum of $184,500, producing $11,253 of OASDI tax rather than $11,439. The remaining $186 gap is 6.2% of the $3,000 understatement of the wage base, with the balance from rounding Medicare tax to $3,606."
-us,scenario_117,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It asserted a $180,600 2026 Social Security wage base when the 2026 taxable maximum is $184,500, capping OASDI tax at $11,197.20 instead of $11,439. Medicare tax of $3,605.79 and the absence of Additional Medicare Tax are correct, so the entire $241.80 shortfall is 6.2% of the $3,900 wage-base error."
-us,scenario_117,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It used $181,800 as the 2026 Social Security wage base rather than the actual taxable maximum of $184,500, yielding $11,271.60 of OASDI tax instead of $11,439. Its Medicare component ($3,605.79) and its no-Additional-Medicare-Tax, no-state-payroll-tax conclusions are correct, so the $167.40 miss is exactly 6.2% × $2,700 of understated wage base."
-us,scenario_117,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It capped Social Security wages at $168,600 — the 2024 taxable maximum, two years stale — instead of the 2026 maximum of $184,500, producing $10,453.20 of OASDI tax rather than $11,439. Medicare at 1.45% × $248,675 = $3,605.79 and the no-Additional-Medicare-Tax conclusion are correct, so the whole $985.80 shortfall is 6.2% of the $15,900 wage-base error."
-us,scenario_117,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It double-counted the $35,525 FLSA overtime premium by adding it to the $248,675 of gross wages, which the prompt states already include overtime pay, inflating the Medicare base to $284,200. Even that method caps out at $15,559.90 with the correct $184,500 taxable maximum; the submitted $18,270 corresponds to no cap-plus-Medicare computation at all and is $3,225.21 above the reference."
-us,scenario_117,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Its answer nets to a Social Security component of $11,327.21 after the correct $3,605.79 Medicare tax, implying a wage base near $182,700 rather than the 2026 Social Security taxable maximum of $184,500. The method is otherwise right — cap the 6.2% tax, apply 1.45% to all wages — so the $111.79 miss is entirely the understated taxable maximum."
-us,scenario_117,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It invented a $172,200 'estimated 2026 limit' for Social Security wages, below even the 2025 maximum of $176,100 and $12,300 below the actual 2026 taxable maximum of $184,500, capping OASDI tax at $10,676.40 instead of $11,439. Medicare ($3,605.79) and the Additional Medicare Tax conclusion are correct, so the $762.60 shortfall is 6.2% of the wage-base error."
-us,scenario_117,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"It applied the full 7.65% FICA rate to all $248,675 of wages with no Social Security wage cap — $248,675 × 0.062 = $15,417.85 plus $3,605.79 of Medicare tax = $19,023.64, matching its $19,023.57 to seven cents of rounding. The 6.2% tax stops at the 2026 taxable maximum of $184,500, so it overstated OASDI tax by $3,978.85."
-us,scenario_117,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Its stated $11,346 Social Security component corresponds to a $183,000 wage base rather than the 2026 taxable maximum of $184,500, which yields $11,439. The Medicare component ($3,605.79) is exact, so the $93 shortfall is precisely 6.2% of the $1,500 by which it understated the taxable maximum."
-us,scenario_117,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Net of the correct $3,605.79 Medicare tax, its $14,880 implies a Social Security component near $11,274, i.e. a wage base of roughly $181,800 instead of the 2026 taxable maximum of $184,500 that produces $11,439. It capped the 6.2% tax at a stale projected base and rounded the total to a whole dollar, missing by $164.79."
-us,scenario_117,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It stated a $181,800 Social Security wage base limit for 2026 when the taxable maximum is $184,500, capping OASDI tax at $11,271.60 instead of $11,439. Medicare tax of 1.45% × $248,675 = $3,605.79 is exact, so the $167.40 miss is 6.2% of the $2,700 wage-base understatement."
-us,scenario_117,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It used a projected $183,900 Social Security wage base rather than the 2026 taxable maximum of $184,500, capping OASDI tax at $11,401.80 instead of $11,439. Every other component is correct — $3,605.79 of Medicare tax, no Additional Medicare Tax under the $250,000 joint threshold, no Arkansas employee payroll tax — leaving a shortfall of exactly 6.2% × $600 = $37.20."
-us,scenario_117,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its $19,028 is the full 7.65% FICA rate applied to all $248,675 of wages with no Social Security wage cap ($19,023.64), rounded. The 6.2% OASDI tax stops at the 2026 taxable maximum of $184,500, giving $11,439 rather than $15,417.85, so it overstated the tax by nearly $4,000."
-us,scenario_117,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It claimed to cap Social Security wages at the 2026 maximum but submitted $22,306.24, which exceeds even an entirely uncapped 7.65% FICA computation on $248,675 ($19,023.64) by $3,282.60. Applying the $184,500 taxable maximum gives $11,439 of OASDI tax plus $3,605.79 of Medicare tax; its figure is consistent with ignoring the cap and inflating the wage base with the already-included $35,525 overtime premium."
-us,scenario_117,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"It applied a flat 7.65% with no Social Security wage cap and assumed head wages that the household facts set to zero; even its own stated method gives $19,023.64 on $248,675, so the round $25,000 is a guess untethered from any computation. Capping the 6.2% tax at the 2026 taxable maximum of $184,500 gives $11,439, which with $3,605.79 of uncapped Medicare tax totals $15,044.79."
-us,scenario_117,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the Head's $8,223 employer-sponsored insurance premium from the Spouse's Medicare wage base, reducing it to $240,452; that premium belongs to a person with no wages and does not reduce another worker's FICA wages, and it applied the reduction only to Medicare while leaving Social Security wages uncapped by it. It also used a $184,000 wage base instead of the 2026 taxable maximum of $184,500, so its $11,408 OASDI tax and $3,487 Medicare tax fall short of $11,439 and $3,605.79."
-us,scenario_117,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the submission carries no payroll_tax entry to evaluate. The correct derivation is $11,439 of employee Social Security tax (6.2% of the $184,500 2026 taxable maximum) plus $3,605.79 of employee Medicare tax (1.45% × $248,675), totaling $15,044.79."
-us,scenario_117,payroll_tax,minimax-m3,llm_error,other,False,"It reported no employee-side payroll tax at all despite the Spouse's $248,675 of gross wages, which are covered employment wages subject to FICA. Employee Social Security tax is 6.2% of the $184,500 2026 taxable maximum = $11,439 and employee Medicare tax is 1.45% of all $248,675 = $3,605.79, so the correct answer is $15,044.79 rather than $0."
-us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It used $176,100 — the 2025 Social Security taxable maximum — as the 2026 wage base, capping OASDI tax at $10,918.20 instead of the $11,439 that the 2026 maximum of $184,500 produces. Its Medicare tax ($3,605.79), Additional Medicare Tax treatment (none below the $250,000 joint threshold), and Arkansas conclusion are correct, so the whole $520.80 gap is 6.2% of the $8,400 wage-base error."
-us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"Its stated components are the reference: $184,500 × 6.2% = $11,439 of Social Security tax, $248,675 × 1.45% = $3,606 of Medicare tax, and no Additional Medicare Tax because combined wages fall below the $250,000 joint threshold — a total of $15,045. It nonetheless submitted $20,603, $5,558 above its own arithmetic, and referenced a phantom '$213,150 of wages' that appears nowhere in the household facts."
+us,scenario_117,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model computed the correct $11,439 SS tax at the $184,500 base, but then switched to a $183,600 base ($14,988.99). It submitted $12,896.44, which matches none of its own calculations. Its final value is disconnected from its correct reasoning."
+us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model added the $35,525 FLSA overtime premium on top of gross wages that already include overtime. It used the 2024 SS wage base of $168,600 instead of $184,500 and applied Additional Medicare Tax at the $200,000 single threshold instead of the $250,000 MFJ threshold. It also counted Arkansas income tax withholding as a payroll tax, then added an invented $1,680 'adjustment'."
+us,scenario_117,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model did not compute FICA and submitted $0, even though the spouse's $248,675 in wages owes $11,439 in Social Security tax (6.2% × $184,500) and $3,605.79 in Medicare tax."
+us,scenario_117,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached the correct $15,044.79 using the $184,500 2026 SS wage base. It then dropped that base for lower guesses ($183,600, $181,800, $179,800) and submitted $14,757.34, which understates SS tax."
+us,scenario_117,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly totaled $11,439 + $3,605.79 = $15,044.79 using the $184,500 wage base. It then submitted $13,347.55 under an unstated 'SS base estimate' that contradicts its own correct calculation."
+us,scenario_117,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 Social Security wage base of $176,100 for 2026 instead of $184,500. This gave SS tax of $10,918.20 instead of $11,439."
+us,scenario_117,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated 2026 SS wage base of $178,200 instead of the actual $184,500. That understates employee SS tax by $390.60 ($11,048.40 vs $11,439)."
+us,scenario_117,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $181,500 SS wage base instead of the 2026 base of $184,500. This gave SS tax of $11,253 instead of $11,439."
+us,scenario_117,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $180,600 SS wage base instead of the 2026 base of $184,500. This gave SS tax of $11,197.20 instead of $11,439."
+us,scenario_117,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $181,800 SS wage base instead of the 2026 base of $184,500. This gave SS tax of $11,271.60 instead of $11,439."
+us,scenario_117,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the 2024 SS wage base of $168,600 instead of the 2026 base of $184,500. This gave SS tax of $10,453.20 instead of $11,439."
+us,scenario_117,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the $35,525 FLSA overtime premium to the $248,675 gross wages and computed Medicare on $284,200. Gross wages already include overtime pay, so this counts the premium twice and inflates the total to $18,270."
+us,scenario_117,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $14,933 implies SS tax of about $11,327, or roughly a $182,700 wage base. It understated the 2026 SS wage base of $184,500, which yields $11,439."
+us,scenario_117,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an estimated $172,200 SS wage base instead of the 2026 base of $184,500. This gave SS tax of $10,676.40 instead of $11,439."
+us,scenario_117,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model's $19,023.57 is about 7.65% of all $248,675 in wages. It applied the 6.2% Social Security rate to the full wages without capping them at the $184,500 wage base."
+us,scenario_117,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's SS tax of $11,346 implies a $183,000 wage base instead of the 2026 base of $184,500. That understates SS tax by $93."
+us,scenario_117,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $14,880 implies SS tax of about $11,274, which matches a $181,800 wage base. The correct 2026 base is $184,500, which yields $11,439."
+us,scenario_117,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $181,800 SS wage base instead of the 2026 base of $184,500. This gave SS tax of $11,271.60 instead of $11,439."
+us,scenario_117,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model projected a $183,900 SS wage base instead of the actual 2026 base of $184,500. This understates SS tax by $37.20 ($11,401.80 vs $11,439)."
+us,scenario_117,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model's $19,028 is about 7.65% of the full $248,675 in wages. It did not cap Social Security wages at the $184,500 2026 wage base."
+us,scenario_117,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $22,306.24 is more than 7.65% of the spouse's total wages ($19,023.64). So it both skipped the $184,500 Social Security cap and used a wage base larger than the $248,675 gross wages. The correct total is $11,439 SS + $3,605.79 Medicare."
+us,scenario_117,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model applied a flat 7.65% to all wages without the $184,500 Social Security cap. It then reported a rounded $25,000 that exceeds even the uncapped 7.65% × $248,675 = $19,023.64."
+us,scenario_117,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the head's $8,223 health premium from the spouse's FICA wages. No pre-tax salary reduction is listed for the spouse, so Medicare applies to the full $248,675. It also used a $184,000 SS base instead of $184,500."
+us,scenario_117,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value or reasoning, so its answer was missing rather than a substantive computation."
+us,scenario_117,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model said there was no employee payroll tax, even though the spouse earns $248,675 in wages. Those wages owe 6.2% SS tax on $184,500 ($11,439) and 1.45% Medicare tax ($3,605.79)."
+us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the 2025 SS wage base of $176,100 for 2026 instead of $184,500. This gave SS tax of $10,918.20 instead of $11,439."
+us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly computed SS tax of $11,439 at the $184,500 base, Medicare of $3,606 and $0 Additional Medicare Tax. Those sum to $15,045, but it submitted $20,603, a number disconnected from its own components."
us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took a roughly $2,340 standard deduction and never itemized the charitable deduction ($25,900 less the 0.5%-of-AGI floor, or $24,646.33), so its taxable income was about $245,805 instead of $223,498.20. It then cut its own ~$9,196 figure to $8,081 with an unexplained 'bracket smoothing' step, and it never applied the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity without the 2026 floor of 0.5% of AGI, giving $222,245 instead of $223,498.20. It then used a 2%/4%/3.9% schedule that produced $8,586.66, well above the $8,189.53 the 2026 schedule gives. It also subtracted only the $145 in personal credits and missed the $60 additional tax credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages even though wages already include it. It also started from federal taxable income, included the state refund, and invented $18,398 in self-employment tax. It then used a 5% top bracket and $1,050 of made-up credits, instead of $223,498.20 of taxable income taxed at $8,189.53 less $205 in credits."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $22,500 of mortgage interest from the $501,000 loan balance, although no interest paid is listed, so it should be $0. That pushed itemized deductions to about $48,400 instead of $24,646.33. It then submitted $11,264, which contradicts its own estimate of about $7,645."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $2,589 state refund as Arkansas income and imputed about $20,040 of mortgage interest when none is listed, putting itemized deductions near $48,000 instead of $24,646.33. Its $11,820 does not follow from its own ~$202,000 taxable income at a 3.9% top rate."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added invented mortgage-interest and medical deductions to charity, which cut taxable income to about $190,000 instead of $223,498.20. Medical expenses fall below the floor, and no mortgage interest is listed. It also subtracted only $145 in credits and missed the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left the $2,589 state refund in Arkansas income and took the full $25,900 in charity without the 0.5%-of-AGI floor, reaching $224,834 of net income instead of $223,498.20. It taxed that under a 3.9%-top schedule to get $8,685, above the $8,189.53 the 2026 schedule gives. It also subtracted only $145 and omitted the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It subtracted the $35,525 FLSA overtime premium as exempt under Arkansas law, but the engine's Arkansas AGI is $248,144.53 with no overtime exclusion. It also added a $10,000 SALT deduction even though no state or local taxes paid are listed, plus $448 of employee business expenses. Finally, it claimed a $240 Arkansas child-care credit even though the federal credit is $0 because the head has no earned income."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It imputed about $20,040 of mortgage interest that is not in the facts, reached a pre-credit tax of about $8,086, and then raised the answer to $12,500 with no computation behind the change. It also never subtracted the $145 in personal credits or the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction and taxed about $245,000, when itemizing the charitable deduction ($24,646.33 after the 0.5%-of-AGI floor) brings taxable income down to $223,498.20. It also applied a 3.9% top rate, which overstates tax compared with the engine's 2026 schedule."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an outdated 4.9% top rate on a taxable income of about $184,000, far below the correct $223,498.20. It also stated that no nonrefundable credits apply, so it dropped the $145 in personal credits and the $60 additional credit. The $8,000 is a rough figure built from these offsetting errors."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $5,000 standard deduction instead of itemizing the charitable deduction ($24,646.33), leaving $243,145 of taxable income instead of $223,498.20. It also used a 3.9%-top schedule and subtracted neither the $145 in personal credits nor the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $185,403.68 of taxable income plus $29,805.33 of deductions implies an AGI of $215,209, which is $250,734 minus the $35,525 overtime premium. That means it treated the FLSA overtime premium as excluded from Arkansas income. The correct Arkansas AGI is $248,144.53, and the correct itemized deduction is $24,646.33, giving $223,498.20 of taxable income."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the outdated 4.4% top rate with 2%/4% lower brackets and a $4,400 standard deduction on $250,734, which still includes the state refund. It never itemized the $24,646.33 charitable deduction and applied no personal credits or the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from federal AGI of $250,734, leaving in the $2,589 state refund that Arkansas excludes. It itemized $26,641, meaning full charity with no 0.5% floor plus a medical amount that does not apply, and taxed $224,093 under a 3.9%-top schedule to get $8,413.63 instead of $8,189.53. It subtracted only $145 and missed the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no working, but $10,560 equals a flat 4.4% on $240,000. That matches the outdated 2023 top rate applied to income with no itemized charitable deduction and no credits, instead of $223,498.20 taxed at $8,189.53 less $205."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"$8,768 equals a flat 3.9% on $224,834, which is $250,734 minus $25,900. That means it kept the state refund in income, took charity without the 0.5%-of-AGI floor, used a flat 3.9% instead of the 2026 schedule, and subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"$8,623 is a flat 3.9% on $224,834 ($8,768) minus $145. So it kept the $2,589 state refund in Arkansas income, skipped the 0.5%-of-AGI charitable floor, used a flat 3.9% instead of the 2026 schedule's $8,189.53, and omitted the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no working, but $10,879 is consistent with the outdated 4.4% top rate applied to about $247,000, meaning income with only a standard deduction. The correct steps are to itemize $24,646.33 in charity, tax $223,498.20 under the 2026 schedule for $8,189.53, and subtract $205 in credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no working, and its $8,234.12 is about $50 above the correct result. The correct result comes from taxing $223,498.20 (AGI $248,144.53 less $24,646.33 of floored charity) at $8,189.53 under the 2026 schedule and then subtracting the full $205 in credits, including the $60 additional credit for qualified individuals. Its figure reflects a higher-rate schedule or only $145 in credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no working, but $9,260 is about 3.9% of $237,000, which is consistent with taxing income after only a standard deduction. The household should itemize $24,646.33 in charity, which brings taxable income to $223,498.20 and pre-credit tax to $8,189.53."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly applied the Arkansas standard deduction instead of itemizing the charitable deduction ($25,900 less the $1,253.67 AGI floor, or $24,646.33), so it taxed about $20,000 too much income. It also used a 3.9% top rate instead of the lower-yielding 2026 schedule."
-us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It reached $222,245 of taxable income (full charity with no 0.5% floor) and then applied an invented bracket table with 3.4%, 4.7% and 4.4% rates to get $8,928.98. It also subtracted neither the $145 in personal credits nor the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It kept the state refund in AGI, added a small medical deduction that the floor rules out, and rounded a 3.9%-top calculation to $8,000. It also stated that no Arkansas nonrefundable credits apply, which drops the $145 in personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It called the Arkansas tax 'modest' and reported $1,005, far too low. The household has $223,498.20 of Arkansas taxable income, which the 2026 schedule taxes at $8,189.53 before the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It reported zero tax by citing unspecified 'high deductions/offsets'. The only Arkansas deduction is $24,646.33 of charity, which leaves $223,498.20 of taxable income and $8,189.53 of tax before the $205 in credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It took the full $25,900 charitable deduction without the 0.5%-of-AGI floor, giving $222,245, and applied a 3.9%-top high-income schedule that produced $8,496.66 instead of $8,189.53. It then subtracted $160 in credits instead of $205 ($145 in personal credits plus the $60 additional credit for qualified individuals)."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used 'standard and personal deductions' instead of itemizing $24,646.33 in charity, which overstates taxable income. It also stated that no nonrefundable credit applies, dropping the $145 in personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It itemized charity but used a graduated schedule that produces more tax than the 2026 schedule's $8,189.53 on $223,498.20. It subtracted only the small personal and dependent credits and left out the $60 additional credit for qualified individuals, which puts it $122 high."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It came within $10.78 but did not reproduce the exact steps: charity of $25,900 less 0.5% of federal AGI ($1,253.67) gives $24,646.33, taxable income is $223,498.20, and pre-credit tax is $8,189.53. The final step subtracts exactly $205 ($145 in personal credits plus the $60 additional credit)."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity without the 0.5%-of-AGI floor, giving $222,245, and taxed that under a 3.9%-top high-income schedule to about $8,578 instead of $8,189.53. It subtracted only $145 and omitted the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"$9,535 plus $145 equals 3.9% of about $248,200, which is essentially the full Arkansas AGI. So it never itemized the $24,646.33 charitable deduction, used a 3.9% rate instead of the 2026 schedule, and omitted the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a 3.9% top marginal rate to taxable income after charity, which overstates the $8,189.53 the 2026 schedule gives on $223,498.20. It also counted only the personal credits and missed the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity from $248,145 with no 0.5%-of-AGI floor and applied a 3.9%-top high-income schedule, giving about $8,585 before credits instead of $8,189.53. It subtracted only $145 and left out the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a round $8,000 with no deduction, bracket or credit steps. The actual result requires $24,646.33 of itemized charity, $223,498.20 of taxable income, $8,189.53 of tax under the 2026 schedule, and $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the outdated 4.4% top rate to about $224,000 of taxable income. The 2026 schedule taxes $223,498.20 at only $8,189.53, and the answer also needs the $145 in personal credits and the $60 additional credit subtracted."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It started from federal AGI of $250,734, keeping the state refund that Arkansas excludes. It added a $741 medical deduction and skipped the 0.5% charitable floor, then taxed $224,093 under a 3.9%-top schedule. It also stated that no nonrefundable credits apply, dropping the $145 in personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used $223,734 as AGI, which is about $24,400 below the true Arkansas AGI of $248,144.53 (wages less the $530 loss). That drove taxable income down to $197,834 instead of $223,498.20. It also missed the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an unexplained AGI of $242,511 instead of $248,144.53 and took charity without the 0.5% floor. It then applied a flat 3.9% instead of the 2026 schedule and subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It taxed about $221,000 at roughly 3.9%, which overstates tax compared with the 2026 schedule's $8,189.53 on $223,498.20. It also subtracted only small personal credits and missed the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so the model produced no usable answer."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took the joint standard deduction, leaving $245,654 of taxable income, instead of itemizing the $24,646.33 charitable deduction, which gives $223,498.20. It also taxed that base under a 3.9%-top schedule and subtracted only $145, omitting the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability. The household has $223,498.20 of Arkansas taxable income taxed at $8,189.53, and only $205 of nonrefundable credits offset it."
-us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the $2,589 state refund in Arkansas AGI and took full charity without the 0.5%-of-AGI floor, reaching $224,834 of taxable income. It then applied a rate of about 3.8% and subtracted $155 in guessed personal credits. That misses the correct $205 in credits ($145 in personal credits plus the $60 additional credit for qualified individuals)."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages, which already include it, and added the state refund, giving an inflated AGI of $286,259. It then taxed income after only a standard deduction at a flat, outdated 4.4%, and dropped the itemized path it had computed itself."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It invented a $28,867 'filing deduction' and subtracted only the $60 dependent credit, dropping the $145 in personal credits. It then submitted $5,105, which contradicts its own stated $7,127."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a ~$2,340 standard deduction, which put taxable income at $245,805. Itemizing the $24,646.33 charitable deduction (after the 0.5% AGI floor) gives $223,498.20 instead. It then lowered its own ~$9,196 result to $8,081 with an unexplained 'bracket smoothing' step and never applied the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its 2%/4%/3.9% high-income schedule gives $8,586.66 on $222,245, which is about $400 more than the reference's $8,189.53 bracket tax on the larger $223,498.20 base. It also deducted the full $25,900 of charity without the 0.5%-of-AGI floor and subtracted only the $145 personal credits, missing the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages, even though gross wages already include it. It also counted the state refund, which Arkansas excludes, and subtracted made-up self-employment taxes and health premiums. It never itemized the charitable deduction, then applied an outdated 5% top bracket and a made-up $1,050 in credits instead of the $205 actual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It entered $0 without computing Arkansas tax. The correct computation is $223,498.20 of taxable income (AGI $248,144.53 minus $24,646.33 of charitable itemized deductions), producing $8,189.53 of tax and $7,984.53 after $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It made up ~$22,500 of mortgage interest from the $501,000 loan balance, although unlisted interest paid must be treated as $0. The only itemized deduction is the $24,646.33 charitable amount after the 0.5% AGI floor. It then reported $11,264, which does not match its own ~$7,645 calculation, and it omitted the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $2,589 state refund in Arkansas income, which Arkansas excludes, and made up ~$20,040 of mortgage interest from the loan balance. It then reported $11,820, far above 3.9% of its own ~$202,000 base. It also missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It cut taxable income to ~$190,000 by adding mortgage interest and medical costs to the charitable deduction. No mortgage interest is listed, and medical costs are below the floor, so the only itemized deduction is the $24,646.33 charitable amount and taxable income is $223,498.20. It also subtracted only $145 of credits instead of $205."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It kept the $2,589 state refund in income and deducted the full $25,900 without the 0.5%-of-AGI floor, giving taxable income of $224,834 instead of $223,498.20. Its schedule produced $8,685 of tax, well above the reference's $8,189.53. It also omitted the $60 additional credit, subtracting only $145."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $35,525 FLSA overtime premium as an Arkansas exclusion, but Arkansas AGI keeps overtime in wages. It also added a made-up $10,000 SALT deduction and a $448 employee-expense deduction, and claimed a $240 Arkansas child-care credit. That credit is $0 because the head has no earnings, so the federal CDCC earned-income limit is $0. It also missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up ~$20,040 of mortgage interest from the $501,000 balance, which is not an interest-paid input, and ignored the 0.5% AGI floor on charitable gifts. It then raised its own ~$8,086 estimate to $12,500 without a basis and never subtracted the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction, taxing ~$245,000. Itemizing the $24,646.33 charitable deduction brings taxable income down to $223,498.20. It also left out the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an outdated 4.9% top rate on an understated ~$184k base instead of the 2026 schedule on $223,498.20. It also stated that no nonrefundable credits apply, so it missed the $145 personal credits and the $60 additional credit; the two errors happened to offset near $8,000."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $5,000 standard deduction (taxable income $243,145) instead of itemizing the $24,646.33 charitable deduction, which gives $223,498.20. It also subtracted none of the $205 in nonrefundable credits (5 × $29 personal credits plus the $60 additional credit)."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $185,403.68 taxable income plus $29,805.33 of deductions comes to $215,209. That means it took the $35,525 overtime premium out of the $250,734 federal AGI and kept the state refund, while Arkansas AGI is $248,144.53 with overtime included and the refund excluded. It also over-itemized and applied none of the $205 in nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied an outdated 4.4% top rate with a 2%/4% lower schedule. It also kept the $2,589 refund in income and took a $4,400 standard deduction instead of itemizing the $24,646.33 charitable deduction. It subtracted none of the $205 nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from $250,734, keeping the state refund that Arkansas excludes, and deducted the full $25,900 of charity plus $741 of medical costs. Its schedule gave $8,413.63 on $224,093, about $224 more than the reference's $8,189.53 bracket tax. It subtracted only $145, missing the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $10,560 equals exactly 4.4% of $240,000. That matches the outdated 4.4% top rate applied to roughly the full AGI without the $24,646.33 charitable itemization, the 2026 schedule ($8,189.53 on $223,498.20), or the $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $8,768 equals 3.9% of $224,834 (the $250,734 federal AGI minus $25,900). That is a flat top rate applied to a base that still contains the state refund and skips the 0.5% charitable floor. It ignored the schedule that produces $8,189.53 and subtracted none of the $205 nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $8,623 equals 3.9% of $224,834 minus $145. That means it kept the state refund in income, skipped the 0.5%-of-AGI charitable floor, and taxed everything at the flat top rate instead of the schedule that gives $8,189.53 on $223,498.20. It also left out the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $10,879 matches an outdated ~4.4% rate applied to nearly all of AGI. It skipped the $24,646.33 charitable itemization that reduces taxable income to $223,498.20, ignored the 2026 schedule that produces $8,189.53, and did not subtract the $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its $8,234.12 sits slightly above the reference's pre-credit tax of $8,189.53. That is consistent with computing the bracket tax on a slightly larger base that keeps the state refund, and then not subtracting the $205 of nonrefundable credits ($145 personal plus $60 additional credit for qualified individuals)."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $9,260 implies taxable income well above $223,498.20. That is consistent with taking a small Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction. It also did not subtract the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used the Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction. That left roughly $22,000 too much taxable income compared with $223,498.20. It also missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied an outdated multi-bracket schedule with 4.7% and 4.4% rates to $222,245 instead of the 2026 schedule, which gives $8,189.53 on $223,498.20. It also deducted the full $25,900 without the 0.5% AGI floor and subtracted none of the $205 nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It stated that no Arkansas nonrefundable credits apply. In fact, $145 of personal credits (5 × $29) plus the $60 additional credit for qualified individuals reduce the $8,189.53 tax to $7,984.53. It also kept the state refund in AGI and added a medical deduction, even though medical costs are below the floor."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It reported only $1,005, as if taxable income were modest. Arkansas taxable income is $223,498.20 (AGI $248,144.53 minus $24,646.33 of charitable itemized deductions), which produces $8,189.53 of tax before $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and offsets wipe out the tax. The only Arkansas itemized deduction is $24,646.33 of charity, which leaves $223,498.20 taxable and $7,984.53 owed after the $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,496.66 on $222,245, more than the reference's $8,189.53 on the larger $223,498.20 base. It also skipped the 0.5%-of-AGI charitable floor and took $160 of credits instead of the $205 actual total ($145 personal plus $60 additional credit)."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the standard deduction instead of itemizing the $24,646.33 charitable deduction, and it stated that no nonrefundable state credit applies. That missed both the $145 personal credits and the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It itemized charity but used a schedule that overstates the bracket tax compared with the reference's $8,189.53 on $223,498.20. It subtracted only the small personal and dependent credits and missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It deducted the full $25,900 of charity without the 2026 floor of 0.5% of AGI ($1,253.67), which cut the tax by about $49. It also subtracted only the $145 dependent and personal credits, omitting the $60 additional credit for qualified individuals. Together these leave it $10.78 high."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,578.26 of tax on $222,245, about $390 above the reference's $8,189.53 on $223,498.20. It also skipped the 0.5% charitable floor and subtracted only $145, missing the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $9,535 implies taxable income of roughly $245k or more. That is consistent with using the standard deduction instead of itemizing the $24,646.33 charitable deduction. It also did not subtract the full $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a ~3.9% top-rate approximation that overstates the bracket tax compared with $8,189.53 on $223,498.20. It also omitted the $60 additional credit for qualified individuals, subtracting only the personal credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,584.76 on $222,245, about $395 more than the reference's bracket tax. It also deducted the full $25,900 without the 0.5% AGI floor and subtracted only $145 of credits, missing the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a rounded $8,000 guess without computing the steps: Arkansas AGI of $248,144.53, the $24,646.33 charitable itemized deduction, $8,189.53 of bracket tax, and $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied an outdated ~4.4% top rate to ~$224,000 instead of the 2026 Arkansas schedule, which gives $8,189.53 on $223,498.20. It also did not subtract the $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"It stated that no nonrefundable Arkansas credits apply, which missed the $145 personal credits and the $60 additional credit for qualified individuals. It also kept the $2,589 refund in AGI and added a $741 medical deduction that does not exist, so its $8,435 exceeds the reference's $8,189.53 pre-credit tax."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It started from a $223,734 AGI, which is $27,000 below the $250,734 federal AGI. That reflects subtracting an overtime-style deduction that Arkansas does not allow, since Arkansas AGI is $248,144.53 with overtime included. The understated base pulled its tax down to $7,491."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $242,511 AGI equals $250,734 minus the head's $8,223 premiums, treating them as an above-the-line deduction. They are not: the coverage is employer-sponsored and there is no self-employment income. It also kept the refund in income, skipped the 0.5% charitable floor, applied a flat 3.9% with no lower brackets, and subtracted none of the $205 credits."
+us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied a flat ~3.9% to ~$221,000, which overstates the bracket tax compared with the reference schedule's $8,189.53 on $223,498.20. It subtracted only small personal credits and missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_income_tax_before_refundable_credits.
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the joint standard deduction, which put taxable income at $245,654, instead of itemizing the $24,646.33 charitable deduction for $223,498.20. It also subtracted only $145 of credits and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability. The household has $223,498.20 of Arkansas taxable income, which produces $8,189.53 of tax and $7,984.53 after the $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the $2,589 refund in AGI, skipped the 0.5% charitable floor, and applied a ~3.8% approximation that overstates the bracket tax. It also used ~$155 of personal credits ($31 × 5) instead of $145 ($29 × 5), and it missed the $60 additional credit for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of gross wages that already include it, and it counted the state refund. It then abandoned its own itemized calculation for a $4,400 standard deduction and applied an outdated flat 4.4% rate, which pushed taxable income to $281,859 instead of $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It used a made-up $213,150 income and a $28,867 'filing deduction'. It counted only the $60 additional credit and left out the $145 personal credits. It then submitted $5,105, which does not match its own $7,127 calculation."
us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model stopped at a generic disability-based Medicaid inquiry and failed to apply New York's automatic SSI-recipient pathway. The head meets SSI's aged criterion and passes the $70 resource and $213.33 countable-income tests, producing a $780.67 monthly SSI benefit and automatic Medicaid eligibility."
+us,scenario_118,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said Social Security income is modest but still concluded 'no Medicaid enrollment assumed; not eligible'. That confuses enrollment with eligibility, and Medicare eligibility does not rule out Medicaid. It also missed that $2,800 a year in Social Security and $70 in assets qualify the head for SSI ($9,368 a year), and in NY that gives automatic Medicaid through the SSI-recipient category."
+us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts were not enough to support eligibility, even though they fully establish the aged/blind/disabled pathway. The head is 74, blind, and disabled, with $2,800 a year in Social Security and $70 in assets. That qualifies them for SSI, which gives automatic NY Medicaid through the SSI-recipient category, and they also pass NY's aged/disabled income and asset tests. The model never applied these tests to the listed income and resources."
us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base."
-us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $233/month of Social Security and never computed the head's $780.67/month of federal SSI ($994 benefit rate minus $213.33 of countable Social Security). SSI is SNAP unearned income, so gross income is $1,014. The model then added a guessed ~$857 utility allowance to reach $0 net income and paid a $293 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month (Jan–Sep) and $248.58 (Oct–Dec)."
-us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly ($1,014 gross) and took the $209 standard deduction, but its shelter costs were only mortgage interest and property tax ($599), giving an excess shelter deduction of $196 and net income of $609. PolicyEngine adds New York's standard utility allowance to shelter costs. That raises the uncapped deduction to about $611 and cuts net income to $194, so the benefit is $239.80/month, not $115."
-us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded that the household exceeds SNAP income limits. In fact the household is categorically eligible through SSI, and its $1,014 gross income (78% of FPG) and $194 net income pass every test. The deductions it listed lower net income and raise the benefit to $298 − $58.20 = $239.80/month; they do not eliminate it."
-us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as the only income, leaving out the $780.67/month of federal SSI the aged, blind and disabled head receives. It also explicitly refused any utility allowance. Counting SSI gives $1,014 gross income, and with the utility allowance in shelter costs, net income is $194. The benefit is therefore $239.80/month, not the $293 maximum it assumed."
-us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and ignored the mortgage-interest and property-tax shelter costs. It then dropped its own ~$3,400 result for an unexplained $2,533. The correct chain is $1,014 gross, minus the $209 standard deduction and the uncapped excess shelter deduction, giving $194 net income and $239.80/month."
-us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the head's $780.67/month of SSI out of gross income and skipped the excess shelter deduction, then miscalculated $298 − 0.3 × $29 as $203 (it is $289). With SSI counted and the uncapped shelter deduction including the utility allowance, net income is $194 and the monthly benefit is $239.80."
-us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI ($1,014 gross) and the $209 standard deduction, but its shelter costs were only the $599 of mortgage interest and property tax, which left net income at $609. PolicyEngine's shelter costs also include New York's standard utility allowance (about $1,013/month in total). That makes the uncapped excess shelter deduction about $611 and net income $194, so the benefit is $239.80/month, not $115."
-us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It stated outright that the head has no SSI, so it counted only $233/month of Social Security and drove net income to $0. An aged, blind and disabled person with $2,800 of Social Security and $70 of assets receives $780.67/month of federal SSI, which SNAP counts as income. It also used the FY2025 $292 maximum instead of FY2026's $298."
-us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, leaving out the $780.67/month of SSI, and reasoned its way to the maximum allotment. It then applied an unexplained cut to reach $2,555. With SSI in gross income ($1,014) and the uncapped shelter deduction, net income is $194 and the benefit is $239.80/month for Jan–Sep and $248.58 for Oct–Dec."
-us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as total income and ignored the head's $780.67/month of federal SSI. That zeroed net income, so it paid the $298 maximum every month. Counting SSI gives about $194 of net income, a $58.20 expected contribution, and a $239.80 monthly benefit."
-us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI. It also took no shelter deduction and used an outdated $199 standard deduction, which produced net income of $34. The correct figure is $194: $1,014 gross, minus $209, minus the uncapped excess shelter deduction."
-us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It set net income to $0 by treating Social Security as the only income, leaving out the $780.67/month of SSI, and it paid the FY2025 $292 maximum. With SSI counted, net income is $194 and the monthly benefit is $298 − $58.20 = $239.80."
-us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI but used only mortgage interest and property tax as shelter costs, giving a $198.75 excess shelter deduction and $601 of net income. PolicyEngine adds New York's standard utility allowance, which makes the deduction about $611 and net income $194. The model also used a $303 maximum allotment instead of FY2026's $298."
-us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly excluded SSI from SNAP income. SSI is countable unearned income for SNAP, so gross income is $1,014, not $233. That mistake zeroed net income and produced a $299 maximum instead of $298 − 0.3 × $194 = $239.80/month."
-us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI at $864.67, above the $780.67 federal SSI that PolicyEngine counts, and assumed a utility allowance big enough (over $735) to wipe out net income. PolicyEngine's total shelter costs are about $1,013/month, which leaves $194 of net income and a $58.20 contribution. The model also used the FY2025 $292 maximum."
-us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without any calculation, claiming the income was too low to produce a benefit. That is backwards: lower income raises SNAP. The household is categorically eligible through SSI, has $194 of net income after deductions, and receives $239.80/month."
-us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $817 (the federal amount is $780.67) and used stale FY2025 parameters: a $198 standard deduction and a $291 maximum. Its shelter costs were only mortgage interest and property tax, which gave $679 of net income. Adding New York's standard utility allowance to shelter costs brings net income to $194 and the benefit to $239.80/month."
-us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied New York's NYSNIP standardized benefit for SSI recipients and paid a flat $291 maximum. PolicyEngine uses the regular SNAP formula: the $298 maximum minus 30% of $194 net income, or $239.80/month ($248.58 in Oct–Dec). $291 is also an outdated maximum."
-us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It declared a $0 benefit on $2,800 of Social Security without doing the calculation. The household is categorically eligible through SSI and has $194 of net income after the standard and uncapped excess shelter deductions, which gives $239.80/month."
-us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Paying the FY2025 $291 maximum all year means it counted only Social Security and zeroed net income. It left out the $780.67/month of SSI, which raises gross income to $1,014 and net income to $194, for a benefit of $239.80/month."
-us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"$828 is $69/month, which implies net income of about $760. That fits counting Social Security plus SSI with a state supplement while deducting, at most, shelter costs made up of mortgage interest and property tax. It left New York's standard utility allowance out of shelter costs. With the allowance included, the uncapped excess shelter deduction is about $611 and net income is $194."
-us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2025 values: SSI of $753.67/month instead of 2026's $780.67, a $200 standard deduction instead of $209, and the $292 maximum. Its shelter costs were only mortgage interest and property tax. Without New York's standard utility allowance, net income stayed at $582/month instead of $194."
-us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $233/month of Social Security and left out the $780.67/month of SSI. That drove net income to $0, and it paid a guessed $295 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month."
-us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and skipped the shelter deduction, which produced $26 of net income. Correct gross income is $1,014; after the $209 standard deduction and the uncapped excess shelter deduction, net income is $194 and the benefit is $239.80."
-us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It denied any benefit, claiming the household has no expenses. That ignores the $5,551 of mortgage interest and $1,634 of property tax, which are SNAP shelter costs, and the SSI that makes the household categorically eligible. The household nets $194/month and receives $239.80/month."
-us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 on the grounds that required facts were missing. The listed Social Security, the resulting SSI, the mortgage interest and the property tax fully determine net income of $194 and a benefit of $239.80/month."
-us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as the only income and applied a $20 exclusion that SNAP does not have. It left out the $780.67/month of SSI and zeroed net income, then used a $243 maximum far below FY2026's $298. The two errors partly cancel; the correct figure is $298 − $58.20 = $239.80/month."
-us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $2,800 of Social Security, leaving out SSI, and reached a near-maximum $290.70/month. With $780.67/month of SSI counted, gross income is $1,014, net income is $194, and the benefit is $239.80/month."
-us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Social Security plus SSI but used only mortgage interest and property tax ($599) as shelter costs, which gave net income of $609 and $115/month. PolicyEngine adds New York's standard utility allowance, which raises the uncapped excess shelter deduction to about $611 and lowers net income to $194. The benefit is $239.80/month."
-us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Social Security plus SSI but used only mortgage interest and property tax as shelter costs, arriving at about $123/month. Leaving out New York's standard utility allowance kept net income near $600 instead of $194, so it missed the $239.80 monthly benefit."
-us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted $1,014 of income and the $209 standard deduction but took no shelter deduction at all. Mortgage interest and property tax are SNAP shelter costs, and the excess shelter deduction is uncapped for elderly/disabled households. With about $1,013 of shelter costs (including the utility allowance), the deduction is about $611, net income is $194, and the benefit is $239.80/month, not $56.50."
-us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the household as having very low income by leaving out the $780.67/month of SSI, and paid the $298 maximum all year. Counting SSI brings net income to $194 and the benefit to $239.80/month for Jan–Sep and $248.58 for Oct–Dec."
-us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added New York's $87 state SSI supplement to income ($1,101/month), but PolicyEngine counts only the $780.67 of federal SSI. Its shelter costs were only mortgage interest and property tax, without the standard utility allowance. Together these left net income at $739/month instead of $194."
-us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly used Social Security plus federal SSI, the $209 standard deduction and the $298 maximum, but its shelter costs were only mortgage interest and property tax, which gave net income of $609 and $115/month. PolicyEngine adds New York's standard utility allowance, which brings net income to $194 and the benefit to $239.80/month."
-us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without doing any calculation. The household is categorically eligible through SSI and nets $194/month after the standard and uncapped excess shelter deductions, which gives $239.80/month."
-us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its ~$735 net income fits counting federal SSI plus New York's state supplement and treating only mortgage interest and property tax as shelter costs. It left out the standard utility allowance that brings net income to $194, and it used a $301 maximum instead of $298."
-us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $0 net income by counting only Social Security, leaving out the $780.67/month of SSI, and paid the FY2025 $292 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month."
-us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly counted $2,800 of Social Security plus $9,368 of SSI and used the $209 standard deduction, but its shelter costs were only the $7,185 of mortgage interest and property tax. That gave $609/month of net income. PolicyEngine adds New York's standard utility allowance, which cuts net income to $194 and raises the benefit to $239.80/month."
-us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI. It set shelter costs to zero despite the listed mortgage interest and property tax, and used 2024 parameters ($193 standard deduction, $291 maximum). The correct figures are $1,014 gross, $194 net and $239.80/month."
-us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as the only income and left out the $780.67/month of SSI, so shelter costs wiped out net income and it paid the $298 maximum all year. Counting SSI leaves $194 of net income and a $239.80 monthly benefit."
-us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, leaving out the $780.67/month of SSI, which zeroed net income. It then used an inflated $309 maximum instead of FY2026's $298. Counting SSI gives $194 of net income and $239.80/month."
-us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI plus New York's $87 state supplement ($1,101/month) and took no shelter deduction, claiming there was no housing-cost input. Mortgage interest and property tax are SNAP shelter costs, and with the utility allowance the uncapped excess shelter deduction is about $611. That leaves $194 of net income and a $239.80 benefit, not $30."
-us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It said $233/month of income is too low to qualify. SNAP has no minimum income requirement, and lower income raises the benefit. The household is eligible through SSI and receives $298 − $58.20 = $239.80/month."
-us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly built $1,014 of gross income, took the $209 standard deduction and applied an uncapped shelter deduction. But its shelter costs were only the $598.75 of mortgage interest and property tax, which gave $608.75 of net income. PolicyEngine adds New York's standard utility allowance, which raises the deduction to about $611, lowers net income to $194, and yields $239.80/month (and $248.58 for Oct–Dec)."
-us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and ignored the mortgage-interest and property-tax shelter deduction. It also used stale parameters ($198 standard deduction, $291 maximum). The correct figures are $1,014 gross, $194 net and $239.80/month."
-us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It required a qualifying child, but SNAP has no child requirement, and a single elderly, disabled SSI recipient is categorically eligible. The household nets $194/month and receives $239.80/month."
-us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"Its reasoning reached the correct federal computation, 12 x ($994 - $213.33) = $9,368, but then dropped it for an incoherent formula (""12*(967-233.33+20)*..."") that mixed in a stray $725.33 FBR. It submitted $6,144, which none of its stated steps support."
-us,scenario_118,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It declared the head income-ineligible because $2,800 of Social Security ""exceeds the SSI income limit."" But $2,800 a year is $233.33 a month, and after the $20 exclusion countable income is $213.33 a month, far below the $994 FBR. The correct federal benefit is $780.67 a month ($9,368 a year), not $0."
-us,scenario_118,ssi,claude-opus-4.7,llm_error,state_local_rule,False,"It used the 2025 FBR of $967 instead of the 2026 FBR of $994. It then added a NY state supplement (about $145/month) to the ssi amount, although ssi is the federal payment only. The correct figure is 12 x ($994 - $213.33) = $9,368."
-us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"Its reasoning computed a positive federal SSI of $8,824 (FBR $11,604 minus $2,780) and called the benefit positive, yet it submitted $0. That reasoning also used the stale $967 FBR and a single annual $20 exclusion instead of $994/month and $240/year."
-us,scenario_118,ssi,claude-opus-5,llm_error,other,False,"It stated a roughly $1,000/month FBR minus $213 countable income plus a NY supplement, yet reported $667/month, an arithmetic result lower than the FBR minus income alone. The correct federal benefit is $994 - $213.33 = $780.67/month, or $9,368 a year, with no state supplement."
-us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 FBR of $967/month instead of $994. It also applied the $20 general exclusion once per year ($2,800 - $20 = $2,780) instead of monthly ($240/year, leaving $2,560 countable), which gave $11,604 - $2,780 = $8,824 instead of $11,928 - $2,560 = $9,368."
-us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"It overstated the FBR at $1,013/month and applied the $20 exclusion only once a year, which gave $9,372. It then cut that to $6,212 by an unexplained ""NY state supplement and rounding"" adjustment. A state supplement never reduces the federal payment, and the correct result is 12 x ($994 - $213.33) = $9,368."
-us,scenario_118,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It computed countable income correctly at $213.33/month but used an FBR of $991 instead of the 2026 value of $994. That gave $777.67/month ($9,332) instead of $780.67/month ($9,368)."
-us,scenario_118,ssi,deepseek-v4-pro,llm_error,state_local_rule,False,"It added an estimated $54/month NY state supplement to the maximum payment, but ssi is the federal SSI payment only. It also used an FBR of $991 instead of $994, which produced ($1,045 - $213.33) x 12 = $9,980 instead of ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed countable income correctly at $213.33/month but used an FBR of $990/month instead of the 2026 value of $994. That gave $776.67/month ($9,320) instead of $780.67/month ($9,368)."
-us,scenario_118,ssi,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It added a NY state supplement of $87.30/month to a $993 FBR, but ssi is the federal SSI payment only. The correct amount is ($994 - $213.33) x 12 = $9,368, not ($1,080.30 - $213.33) x 12 = $10,404."
-us,scenario_118,ssi,gemini-3-flash-preview,llm_error,state_local_rule,False,"It added an $87/month NY living-alone supplement to a $991 FBR, but ssi is the federal SSI payment only. It also understated the 2026 FBR, which is $994. The correct amount is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,period_annualization,False,"It claimed the head's Social Security exceeds the federal maximum benefit rate. That is only true if $2,800 of annual income is compared against a monthly FBR. Monthly Social Security is $233.33, so countable income of $213.33 leaves a federal payment of $780.67/month ($9,368/year)."
-us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added a $1,044 NY state supplement to the federal maximum, but ssi is the federal SSI payment only. It also used the 2024 FBR ($11,316/year, $943/month) instead of the 2026 FBR of $11,928, giving $9,800 instead of $11,928 - $2,560 = $9,368."
-us,scenario_118,ssi,gemini-3.5-flash,llm_error,state_local_rule,False,"It explicitly used the 2024 FBR of $943/month instead of the 2026 FBR of $994. It then added an $87/month NY state supplement, which is outside the federal ssi amount. The correct figure is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,gemini-3.5-flash-lite,llm_error,period_annualization,False,"It compared $2,800 of annual Social Security against the monthly FBR and concluded that income exceeds it. On a monthly basis, countable income is $233.33 - $20 = $213.33, well under the $994 FBR, for a federal payment of $9,368/year."
-us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560, but its $9,236 answer implies an annual FBR of $11,796 ($983/month) instead of the 2026 FBR of $11,928 ($994/month)."
-us,scenario_118,ssi,gemini-3.7-flash,llm_error,state_local_rule,False,"It explicitly combined the federal payment with a NY state supplement, but ssi is the federal SSI payment only. Its $10,376 plus $2,560 of countable income implies a $991 FBR plus $87 of SSP, instead of the $994 federal-only rate that gives $9,368."
-us,scenario_118,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560 but used the 2025 FBR of $967/month ($11,604/year) instead of the 2026 FBR of $994/month ($11,928/year). That gave $9,044 instead of $9,368."
-us,scenario_118,ssi,glm-5.2,llm_error,state_local_rule,False,"It added a $94/month NY supplement for a blind individual to the federal payment, but ssi is the federal SSI payment only. It also projected the 2026 FBR at $991 instead of $994. The correct amount is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the listed facts do not establish SSI. But the head is 74, blind and disabled, has $70 in resources (under the $2,000 limit), and has only $213.33/month of countable income, so the head qualifies for a federal payment of $780.67/month ($9,368/year)."
-us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted that no age or disability facts were provided, although the household lists age 74, blind and disabled. It never applied the income test: $994 FBR minus $213.33 of countable Social Security is $9,368/year."
-us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"It used a sub-2026 FBR (stated as $982.67/month) instead of the 2026 FBR of $994/month ($11,928/year). Its $9,332 corresponds to an $11,892 annual FBR ($991/month), which does not match its own stated rate, instead of $11,928 - $2,560 = $9,368."
-us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"It had the correct $11,928 annual FBR but applied the $20 general exclusion once per year ($2,800 - $20 = $2,780). The exclusion is $20 per month, which leaves $2,560 of countable income, so the benefit is $9,368 instead of $9,148."
-us,scenario_118,ssi,gpt-6-sol,llm_error,state_local_rule,False,"It correctly used the $11,928 FBR and $2,560 of countable income, but then added $1,044 of NY state supplementation. The ssi output is the federal payment alone, $9,368."
-us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"It claimed there were no inputs to calculate SSI, ignoring the listed aged (74), blind and disabled status, the $70 of resources and the $2,800 of Social Security. Those facts give a federal SSI of ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,grok-4.5,llm_error,state_local_rule,False,"It added an $87/month NY living-alone SSP to a $991 FBR, but ssi is the federal SSI payment only. The 2026 FBR is also $994. The correct amount is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,grok-4.6,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560, but its $9,044 answer implies the 2025 FBR of $11,604 ($967/month) instead of the 2026 FBR of $11,928 ($994/month)."
-us,scenario_118,ssi,grok-build-0.1,llm_error,state_local_rule,False,"It used the 2024 FBR of $943 as a proxy for 2026 instead of the 2026 FBR of $994. It then added an $87/month NY supplement, which is outside the federal ssi amount. The correct figure is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,ssi,inkling,llm_error,state_local_rule,False,"It correctly computed a federal payment of about $780/month from the $994 FBR, then added an $87/month NY state supplement. The ssi output excludes the state supplement and equals $780.67 x 12 = $9,368."
-us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560 but used an estimated FBR of $991/month ($11,892/year) instead of the 2026 FBR of $994/month ($11,928/year). That gave $9,332 instead of $9,368."
-us,scenario_118,ssi,kimi-k3,llm_error,state_local_rule,False,"It correctly derived $9,368 of federal SSI ($11,928 - $2,560), then added a $1,044 NY living-alone supplement. The ssi output is the federal payment only."
-us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"It asserted that $233/month of Social Security exceeds a $967+/month FBR, which reverses the comparison. Countable income of $213.33/month is far below the $994 FBR, which leaves a federal payment of $780.67/month ($9,368/year)."
-us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"Its reasoning computed a positive federal SSI of $8,756/year, then submitted $0, contradicting its own calculation. That calculation also used the 2024 FBR of $943 instead of the 2026 FBR of $994, which gives $9,368."
-us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"It declared SSI not positive without any income or resource test, despite the listed aged, blind and disabled status, $70 of resources and only $213.33/month of countable income. The correct federal payment is ($994 - $213.33) x 12 = $9,368."
-us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said homeowners with these facts don't qualify for a circuit-breaker payment, but the NY Real Property Tax Credit covers owner-occupants who pay real property taxes. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a 74-year-old gets a flat $375."
-us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model looked only at the Empire State EITC and concluded NY has no other refundable credit. It never applied the refundable Real Property Tax Credit, which gives this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum."
-us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified the RPTC and the $375 senior maximum, then cut it to an unexplained $63. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375, so nothing reduces it to $63."
-us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model assumed a refund from the real property tax credit requires NY tax liability. The RPTC is refundable and is paid even when NY taxable income is zero, giving $375 to this 65+ homeowner."
-us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the real property tax credit as needing rent or taxable income. Real property taxes paid by a homeowner qualify directly, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375."
-us,scenario_118,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula, getting $452 as 50% of the property taxes above 6% of a $12,168 household gross income. It then capped that at an income-banded $171, but a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375."
-us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula to get $733 and then applied a $75 maximum it tied to having no dependents. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the flat $375 for a filer 65 or older applies."
-us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model dismissed the senior circuit breaker as negligible and tied it to rent. For homeowners, the RPTC is based on real property taxes paid, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, this 65+ filer receives a flat $375."
-us,scenario_118,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model reported the NY household credit, which is nonrefundable and not part of state refundable credits, and explicitly skipped the real property tax credit. The refundable RPTC for this 65+ homeowner is $375."
-us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed zero state tax liability rules out refundable credits. It missed the refundable NY Real Property Tax Credit, which pays this 74-year-old homeowner the $375 elderly maximum."
-us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used the repealed household-gross-income rule to reach $375, the right RPTC for a filer 65 or older with federal AGI of $0, but added a $75 NY household credit. That credit is nonrefundable and is not part of state refundable credits, so the total is $375, not $450."
-us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said the household doesn't qualify for any NY refundable credit. It missed the refundable Real Property Tax Credit, which pays a 65+ homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a flat $375."
-us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model checked only the Empire State Child Credit and the NY EITC. It never applied the refundable Real Property Tax Credit, which needs neither earned income nor dependents and pays this 74-year-old homeowner $375."
-us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model's $0 leaves out the refundable NY Real Property Tax Credit. For a 74-year-old homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, the credit is a flat $375."
-us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model's $0 leaves out the refundable NY Real Property Tax Credit. It applies here: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375.
-us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model assumed refundable credits require earned income or dependents. The NY Real Property Tax Credit requires neither and pays this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum."
-us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,The model ruled out only the Empire State Child Credit and the NY EITC and never checked the refundable Real Property Tax Credit. That credit gives this low-income 65+ homeowner $375.
-us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula, getting $768 (50% of $1,634 minus 3.5% of $2,800), and compared it to a $1,000 senior maximum that does not apply here. A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375."
-us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model checked only the NY EITC. It missed the refundable Real Property Tax Credit, which needs no earned income and pays this 74-year-old homeowner the $375 elderly maximum."
-us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said no NY refundable credit applies. The listed facts (age 74, owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0) qualify for the refundable Real Property Tax Credit at a flat $375 for a filer 65 or older."
-us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model concluded no refundable credit applies and missed the NY Real Property Tax Credit. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375."
-us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated income consisting only of Social Security as ruling out any NY refundable credit. The Real Property Tax Credit still applies to this 74-year-old homeowner with federal AGI of $0 and property taxes above 3.5% of it, and pays a flat $375."
-us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model reported the $90 NY household credit, which is nonrefundable and not part of state refundable credits. It left out the refundable Real Property Tax Credit, which is $375 for this 65+ homeowner."
-us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model identified the RPTC but used the repealed household-gross-income rule, with an estimated $12,168 of household gross income, and capped the credit at $75 for a homeowner 65 or older. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the credit is a flat $375."
-us,scenario_118,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model used an invented formula of 2.5% of real estate taxes. From 2025 the RPTC is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives $375."
-us,scenario_118,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model applied an income-banded $171 cap to the elderly RPTC. A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375."
-us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model found no qualifying credits and missed the refundable NY Real Property Tax Credit. It pays this 74-year-old homeowner, whose $1,634 in property taxes is above 3.5% of a federal AGI of $0, a flat $375."
-us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model treated the NY EITC and the Empire State Child Credit as the only refundable credits. It left out the refundable Real Property Tax Credit, which is $375 for this 65+ homeowner."
-us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model asserted that property tax credits don't apply. A 74-year-old owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0, qualifies for the refundable RPTC at a flat $375."
-us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked the EITC, the child credit, and the tuition and childcare credits, but never the refundable Real Property Tax Credit. The household's real estate taxes and the head's age of 74 qualify it for the $375 maximum."
-us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model assumed that having no NY tax liability rules out refundable credits. The Real Property Tax Credit is paid without liability and gives this 65+ homeowner $375.
-us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model considered only the NY EITC and the child and dependent care credit, both of which need earnings. It missed the refundable Real Property Tax Credit, which is based on property taxes paid and gives this 74-year-old $375."
-us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model found no eligibility factors, overlooking that a 74-year-old homeowner with federal AGI of $0 and $1,634 in real estate taxes, above 3.5% of that AGI, receives the refundable RPTC at a flat $375."
-us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model cited the head's income and age as reasons for no credit, but they are what qualify the head for the Real Property Tax Credit: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375."
+us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $233/month of Social Security and left out the $780.67/month of SSI the head receives, so it understated SNAP gross income, which is $1,014. With that low income and an assumed $857 utility allowance, net income went to $0 and it paid a $293 maximum. The correct net income is $194.38, which gives $239.80/month for most of the year."
+us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted SS plus SSI ($1,014) and the $209 standard deduction. But it built shelter costs from mortgage interest and property tax alone ($599) and left out the utility allowance PolicyEngine adds, which brings shelter costs to about $1,013. That left net income at $609 instead of $194.38. It also missed the October–December FY2027 increase to $248.58/month."
+us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Concluded the household exceeds SNAP income limits even though its own reasoning describes very low income and large shelter deductions. The household is categorically eligible through SSI and easily passes the net income test ($194.38 against $1,304 FPG), so its $0 contradicts the $239.80/month allotment."
+us,scenario_118,snap,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 without doing a calculation. It missed that this one-person elderly/disabled household is categorically eligible through SSI. With $1,014 gross income and the standard and uncapped shelter deductions, it gets $239.80–$248.58/month."
+us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Treated $233/month of Social Security as the only income and left out the $780.67/month of SSI that SNAP counts as unearned income. That drove net income to $0 and the benefit to a $293 maximum. With SSI counted, net income is $194.38 and the allotment is $239.80/month."
+us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI and ignored the listed shelter costs, so it computed net income of only $29. Its final $2,533 also doesn't match its own figure of about $283/month (about $3,400/year)."
+us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only Social Security, leaving out the $780.67/month of SSI, and skipped the shelter deduction. It also made an arithmetic error: $298 − 0.3 × $29 is $289, not the $203 it used."
+us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS plus SSI ($1,014) and the $209 standard deduction correctly. But it limited shelter costs to the $599 of mortgage interest and property tax, without the utility allowance PolicyEngine includes. The result was net income of $609 instead of $194.38 and a $115 benefit instead of $239.80."
+us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly assumed the head gets no SSI. In fact $233/month of Social Security against the $994 SSI benefit rate leaves an SSI payment of $780.67/month, which SNAP counts as income. Leaving it out zeroed net income, and it then used the FY2025 $292 maximum instead of $298."
+us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Left the $780.67/month of SSI out of gross income and concluded that net income was near zero. Its unexplained reduction from a $3,504 maximum to $2,555 doesn't follow from any benefit-reduction calculation."
+us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted income as only $233/month, leaving out the SSI that brings SNAP gross income to $1,014. As a result net income fell to $0 and it gave the $298 maximum. Correct net income is $194.38, which leaves a $58.20 expected contribution."
+us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,Left out the $780.67/month of SSI from countable income and also skipped the excess shelter deduction for the listed mortgage interest and property tax. It used a $199 standard deduction instead of $209 and ended up near the maximum benefit.
+us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Did not count the SSI payment ($780.67/month) as SNAP income, so net income came out to $0. It then applied the FY2025 $292 maximum, which overstates the benefit compared with the $239.80/month that follows from $194.38 net income."
+us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI, but it computed the excess shelter deduction ($198.75) from mortgage interest and property tax alone, without the utility allowance. That left net income at $601 instead of $194.38. It also used a $303 maximum and a $210 standard deduction instead of $298 and $209."
+us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly excluded SSI from SNAP income, but SSI is countable unearned income under 7 CFR 273.9(b)(2). Leaving out $780.67/month drove net income to zero, and it then used a $299 maximum allotment."
+us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $864.67 (the correct amount is $780.67). It also assumed a utility allowance large enough to wipe out all net income. PolicyEngine's total shelter costs of about $1,013 leave $194.38 of net income, and the correct maximum is $298, not $292."
+us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,Returned $0 without a benefit calculation. It missed that the household is categorically eligible through SSI and that $194.38 of net income yields $239.80/month.
+us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $817 (correct: $780.67) and used outdated values: a $198 standard deduction instead of $209 and a $291 maximum instead of $298. It limited shelter costs to mortgage interest and property tax, so net income came out at $679 instead of $194.38."
+us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the NYSNIP pathway grants the maximum benefit automatically. PolicyEngine applies the regular SNAP formula, which subtracts 30% of the $194.38 net income (including SSI) from the $298 maximum. It also used a stale $291 maximum."
+us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Asserted a $0 benefit with no calculation and counted only the $2,800 of Social Security. This one-person SSI recipient has net income of $194.38 and gets $239.80–$248.58/month."
+us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only Social Security, leaving out the $780.67/month of SSI. That zeroed net income, and it then applied a stale $291 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $828/year ($69/month) implies net income of about $760/month, which means it took almost no excess shelter deduction against the $805 of adjusted income. PolicyEngine's shelter deduction brings net income down to $194.38."
+us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Understated SSI at $9,044 (correct: $9,368) and used outdated values: a $200/month standard deduction instead of $209 and a $292 maximum instead of $298. It also computed excess shelter from mortgage interest and property tax only, without the utility allowance, so net income came out at $582/month instead of $194.38."
+us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Started from $233.33/month of Social Security and left out the $780.67/month of SSI SNAP counts. Net income therefore became $0, and it used a guessed $295 maximum instead of $298 minus a $58.20 contribution."
+us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the SSI income and also ignored the listed mortgage interest and property tax shelter costs. It reduced the maximum by 30% of only $26 of net income, using guessed values ($207 standard deduction, $296 maximum)."
+us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed no expenses were listed, although mortgage interest and property tax are, and returned $0. The household is categorically eligible through SSI and, after the standard and uncapped shelter deductions, has $194.38 of net income, which yields $239.80/month."
+us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 on the grounds that the facts were insufficient. The listed income, assets, and shelter costs are enough to compute a positive SNAP allotment for this SSI-eligible elderly/disabled household."
+us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the SSI program's $20 general income exclusion as if it were a SNAP deduction and left SSI out of SNAP income, so net income came out to zero. It also used a $243 maximum allotment instead of $298."
+us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $2,800 of Social Security and only the standard deduction, leaving out SSI income and the shelter deduction. The result was a near-maximum $290.70/month instead of $239.80."
+us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS plus SSI correctly, but it built the excess shelter deduction from mortgage interest and property tax alone, without the utility allowance PolicyEngine adds. That left net income near $609 instead of $194.38 and produced $115/month instead of $239.80."
+us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS and SSI but limited shelter costs to the listed mortgage interest and property tax, without the utility allowance. Its $123/month implies net income near $583, against PolicyEngine's $194.38."
+us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted $1,014 of SS plus SSI and the $209 standard deduction. But it explicitly took no shelter deduction, even though mortgage interest and property tax are listed and the elderly/disabled excess shelter deduction is uncapped. That left net income at $805 instead of $194.38."
+us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Treated income as very low by leaving out the $780.67/month of SSI, so it assigned the $298 maximum. With SSI counted, net income is $194.38 and the 30% contribution is $58.20/month."
+us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year (correct: $9,368) and computed only a $1,833 excess shelter deduction from mortgage interest and property tax, without the utility allowance. That gave net income of $739/month instead of $194.38."
+us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used the right $298 maximum, $209 standard deduction, and SS plus SSI income. But its shelter costs were only mortgage interest and property tax ($598.75), without the utility allowance. That gave net income of $608.75 instead of $194.38."
+us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 on the claim that the inputs were missing. It never applied SSI categorical eligibility or the SNAP formula, which gives $239.80/month for January–September and $248.58/month for October–December."
+us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,Identified SSI categorical eligibility but estimated net income at about $735/month. That is far above the $194.38 that results from adding the utility allowance to mortgage interest and property tax in the uncapped shelter deduction. It also used a $301 maximum instead of $298.
+us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left the SSI payment out of countable income, so net income became $0. It then applied the FY2025 $292 maximum instead of $298 minus the $58.20 contribution."
+us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted SS of $2,800 plus SSI of $9,368 and the $209 standard deduction. But it ran the uncapped excess shelter calculation on only $7,185 of mortgage interest and property tax, without the utility allowance. That gave net income of $608.75/month instead of $194.38."
+us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed shelter costs were unlisted, although mortgage interest and property tax are given. It also left out the $780.67/month of SSI and used 2024 values ($193 standard deduction, $291 maximum)."
+us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,Counted only the $233/month of Social Security and left out the $780.67/month of SSI. That drove net income to $0 and gave the $298 maximum instead of $239.80/month.
+us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left the SSI payment out of SNAP gross income, so net income came out to $0. It then used an overstated $309 maximum instead of $298."
+us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year (correct: $9,368). It also claimed no housing input was supplied and skipped the excess shelter deduction, even though mortgage interest and property tax are listed. That inflated net income to $892/month."
+us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Reasoned that income was 'too low to qualify', which inverts the SNAP formula: lower net income raises the allotment. This SSI-eligible household gets $298 minus 30% of $194.38 of net income."
+us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Counted $1,014 of income and the $209 deduction correctly. But it set shelter costs at only the $598.75 of mortgage interest and property tax, without the utility allowance PolicyEngine adds. That gave an excess shelter deduction of $196.25 instead of about $611 and net income of $608.75 instead of $194.38."
+us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,Left out the $780.67/month of SSI and never applied the shelter deduction for the listed mortgage interest and property tax. It reduced only a stale $291 maximum by 30% of $35.
+us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Treated SNAP as requiring a qualifying child, but single-adult households are fully eligible. This elderly/disabled SSI recipient qualifies for $239.80–$248.58/month."
+us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model's own reasoning correctly computed 12 × ($994 − $213.33) = $9,368. It then dropped that result, cited a made-up $725.33 FBR and an incoherent adjustment formula, and submitted $6,144. That number comes from no valid SSI computation."
+us,scenario_118,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model compared the $2,800 annual Social Security total against a monthly FBR and concluded that income exceeds the limit. In fact, countable income is only $213.33/month after the $20 exclusion, well below the $994 2026 FBR. It also used the stale 2024 FBR of $943."
+us,scenario_118,ssi,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model asserted that $2,800/year of Social Security exceeds the SSI income threshold. Monthly countable income is $233.33 − $20 = $213.33, far below the $994 FBR, so the federal payment is $780.67/month ($9,368/year)."
+us,scenario_118,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025 FBR of $967 instead of the 2026 FBR of $994. It also added an estimated NY state supplement, which is not part of the federal `ssi` output, and inflated the total to $10,788 instead of 12 × ($994 − $213.33) = $9,368."
+us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"The model's reasoning found a positive payment ($8,824) but it submitted 0. That figure was also wrong: it used the 2025 FBR of $967 instead of $994 and subtracted the $20 general exclusion once per year ($2,780) instead of $20 per month ($2,560)."
+us,scenario_118,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate FBR of about $1,000 instead of $994 and folded in a NY state supplement that does not belong in federal SSI. Its arithmetic is internally inconsistent: $1,000 − $213 plus a supplement cannot yield $667/month. The correct federal payment is $994 − $213.33 = $780.67/month."
+us,scenario_118,ssi,claude-sonnet-4.6,llm_error,period_annualization,False,"The model applied the $20 general income exclusion once per year ($2,800 − $20 = $2,780) instead of monthly ($240/year, leaving $2,560 countable). It also used the 2025 FBR of $967 instead of the 2026 FBR of $994, which produced $8,824 instead of $9,368."
+us,scenario_118,ssi,claude-sonnet-5,llm_error,thresholds_rates,False,"The model overestimated the 2026 FBR at $1,013/month instead of $994. It also subtracted the $20 exclusion only once per year. It then replaced its own $9,372 result with an unexplained 'NY supplement/rounding' figure of $6,212 that matches no SSI formula."
+us,scenario_118,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The income counting was correct ($213.33/month countable), but the model used an estimated 2026 FBR of $991 instead of the actual $994. That gives $9,332 instead of 12 × $780.67 = $9,368."
+us,scenario_118,ssi,deepseek-v4-pro,llm_error,state_local_rule,False,"The model added a $54/month NY state supplement to the federal benefit rate, but the `ssi` output is federal SSI only. It also used a $991 FBR instead of the 2026 FBR of $994."
+us,scenario_118,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a 2026 FBR of $990/month instead of $994. Its countable income of $213.33/month was correct, so the result was $9,320 instead of $9,368."
+us,scenario_118,ssi,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model added an $87.30/month NY state supplement to a $993 FBR. The `ssi` output is federal SSI only, computed from the $994 FBR minus $213.33 of countable income, which gives $9,368."
+us,scenario_118,ssi,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model combined a $991 FBR with an $87 NY living-alone supplement. The federal `ssi` output excludes the state supplement and uses the $994 2026 FBR, which yields $780.67/month."
+us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model claimed Social Security income exceeds the federal benefit rate, which compares $2,800 annual income against a monthly FBR. Countable income is $213.33/month, far below the $994 FBR, so the head receives $780.67/month."
+us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Countable income of $2,560 was correct. However, the model used the 2024 FBR ($943/month, $11,316/year) instead of the 2026 FBR of $994 ($11,928/year), and it added a $1,044 NY state supplement that is not part of federal SSI."
+us,scenario_118,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,The model explicitly used the 2024 FBR of $943 for a 2026 calculation instead of $994. It also added an $87/month NY living-alone supplement that is not part of the federal `ssi` output.
+us,scenario_118,ssi,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model treated $2,800 of annual Social Security as exceeding the monthly FBR. Countable income is only $213.33/month against a $994 FBR, which leaves a $780.67 monthly payment."
+us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"Countable income of $2,560 was correct, but the $9,236 answer implies an FBR of $983/month ($11,796/year) instead of the 2026 FBR of $994 ($11,928/year)."
+us,scenario_118,ssi,gemini-3.7-flash,llm_error,state_local_rule,False,"Countable income of $2,560 was correct, but $10,376 implies a combined $1,078/month rate: a $991 FBR plus an $87 NY state supplement. The federal `ssi` output uses only the $994 FBR, with no state supplement."
+us,scenario_118,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 FBR of $967/month ($11,604/year) instead of the 2026 FBR of $994 ($11,928/year). Its countable income of $2,560 was correct, so the error is entirely in the stale benefit rate."
+us,scenario_118,ssi,glm-5.2,llm_error,state_local_rule,False,The model added an estimated $94/month NY supplement for a blind individual on top of a $991 FBR. The `ssi` output excludes the state supplement and uses the $994 2026 FBR.
+us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model said the facts do not establish a positive SSI benefit, ignoring that the head is aged (74) and blind with $70 in resources. The income test was never run: $213.33/month of countable income against the $994 FBR leaves $780.67/month."
+us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed no SSI eligibility facts were provided, even though the household lists age 74, blindness, disability, and $70 in resources. Those facts satisfy both the categorical and resource tests, and the income test leaves a $9,368 annual benefit."
+us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"Countable income of $2,560 was correct, but the model understated the 2026 FBR. Its stated $982.67/month would give $9,232, and its submitted $9,332 corresponds to $991/month. Neither matches the actual $994 FBR."
+us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model used the correct $11,928 annual FBR but subtracted the $20 general exclusion only once per year ($2,780 countable). The exclusion is $20 per month, or $240/year, which leaves $2,560 countable and a $9,368 benefit."
+us,scenario_118,ssi,gpt-6-sol,llm_error,state_local_rule,False,"The model correctly computed federal SSI as $11,928 − $2,560 = $9,368. It then added a $1,044 NY state supplement, which is not part of the federal `ssi` output."
+us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model ran no SSI computation and defaulted to 0, even though the head is aged and blind, holds $70 in resources, and has only $213.33/month of countable income against the $994 FBR."
+us,scenario_118,ssi,grok-4.5,llm_error,state_local_rule,False,"The model added an $87/month NY living-alone supplement to a $991 FBR. The federal `ssi` output excludes the state supplement and uses the $994 2026 FBR, which gives $780.67/month."
+us,scenario_118,ssi,grok-4.6,llm_error,thresholds_rates,False,"Countable income of $2,560 was correct, but $9,044 implies the 2025 FBR of $967/month ($11,604/year) instead of the 2026 FBR of $994 ($11,928/year)."
+us,scenario_118,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the 2024 FBR of $943 as a proxy for 2026 instead of $994. It also added an $87 NY supplement, which is not part of the federal `ssi` output."
+us,scenario_118,ssi,inkling,llm_error,state_local_rule,False,"The model correctly used the $994 FBR and $213 of countable income to get about $780/month of federal SSI. It then added an $87/month NY state supplement, which the federal `ssi` output excludes."
+us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"Countable income of $2,560 was correct, but the model used an estimated 2026 FBR of $991/month ($11,892/year) instead of $994 ($11,928/year), which gave $9,332 instead of $9,368."
+us,scenario_118,ssi,kimi-k3,llm_error,state_local_rule,False,"The model correctly computed federal SSI of $9,368 from the $11,928 FBR minus $2,560. It then added a $1,044 NY living-alone supplement, which is not part of the federal `ssi` output."
+us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"The model stated that $233/month of Social Security exceeds a $967+ FBR, which is an inverted comparison because $233 is far below the FBR. Countable income of $213.33 against the $994 FBR leaves $780.67/month."
+us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model's reasoning computed a positive $8,756 benefit but it submitted 0 with no basis. That figure also relied on the stale 2024 FBR of $943 instead of the 2026 FBR of $994."
+us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model acknowledged that the head is blind and disabled but returned 0 without running the income test. Countable income of $213.33/month against the $994 FBR yields $780.67/month, or $9,368/year."
+us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Wrongly said homeowners with these facts don't qualify for the circuit-breaker credit. The NY real property tax credit applies to homeowners who pay real property taxes and have household gross income of $18,000 or less. For a 65+ filer whose $1,634 in taxes far exceeds the income threshold, it pays the $375 elderly maximum even with zero NY tax."
+us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said NY has no general refundable credit that applies here and only looked at the Empire State EITC. It missed the refundable real property tax credit (IT-214), which gives this 74-year-old low-income homeowner the $375 elderly maximum because of the $1,634 in property taxes."
+us,scenario_118,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Made refundable credits depend on having NY tax liability, earned income, or children. The real property tax credit depends only on property taxes paid, age, and household gross income of $18,000 or less, so this homeowner gets the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Correctly picked the senior real property tax credit and its $375 maximum, then gave an unexplained ~$63. The $1,634 in property taxes is far above the small income-based percentage of near-zero household income, so the cap binds and the credit is the full $375."
+us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Wrongly said the real property tax credit needs tax liability to produce a refund. The credit is fully refundable and is paid with zero NY tax, so this 74-year-old homeowner gets $375."
+us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the lack of rent as ruling out the real property tax credit. Homeowners qualify through the real property taxes they pay, and $1,634 in taxes gives this 65+ filer the $375 maximum credit."
+us,scenario_118,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Counted $2,800 of Social Security plus about $9,368 of estimated SSI as household gross income ($12,168). That puts the 65+ filer in the $12,001–$13,000 band with a $171 cap. The engine's household income measure for this credit leaves out these nontaxable benefits, so the filer falls in the lowest band and gets the full $375 elderly maximum."
+us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly found the real property tax credit and that the computed credit is above the cap, but used the $75 maximum for filers under 65. The cap depends on whether a household member is 65 or older, not on dependents. For this 74-year-old the elderly maximum is $375."
+us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Dismissed the real property tax credit as negligible and tied it to paying rent. A homeowner paying $1,634 in property taxes with almost no income qualifies, and the 65+ cap gives $375."
+us,scenario_118,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Reported the NY household credit, which is nonrefundable and not part of state refundable credits. It also explicitly skipped the real property tax credit, which is the only refundable credit here and is worth $375."
+us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Assumed zero state tax liability rules out refundable credits. The NY real property tax credit is refundable regardless of liability and pays this 65+ homeowner $375.
+us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Correctly got the $375 real property tax credit, then added a $75 NY household credit. The household credit is nonrefundable and only reduces tax liability, which is zero, so it adds nothing to refundable credits. The total stays at $375."
+us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Said no NY refundable credit applies and never considered the real property tax credit. A 74-year-old homeowner with $1,634 in property taxes and household gross income under $18,000 gets the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Only checked the Empire State Child Credit and state EITC. It missed the refundable real property tax credit, which doesn't need earned income or dependents and pays this 65+ homeowner $375."
+us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Said no state refundable credit applies and overlooked the NY real property tax credit. The household's $1,634 in property taxes and the head's age of 74 qualify it for the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Overlooked the refundable NY real property tax credit for low-income homeowners. With $1,634 in property taxes and a 74-year-old head, the credit is the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Required earned income or dependents for any refundable credit. The NY real property tax credit needs neither and pays this 65+ homeowner $375 based on $1,634 in property taxes."
+us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Only ruled out the Empire State Child Credit and NY EITC, and never evaluated the real property tax credit. That credit pays this 74-year-old homeowner the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly found the real property tax credit and computed 50% of the taxes above 3.5% of income ($768), but applied a made-up $1,000 'expanded' maximum. The statutory cap for a 65+ household in the lowest income band is $375, so the credit is $375."
+us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Only checked the NY EITC and missed the refundable real property tax credit. A 74-year-old homeowner paying $1,634 in property taxes gets the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Said nothing triggers a NY refundable credit and missed the real property tax credit. The head's age (74), homeownership with $1,634 in property taxes, and household gross income under $18,000 qualify for the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Never evaluated the NY real property tax credit, the refundable circuit-breaker credit for low-income homeowners. With $1,634 in property taxes and a head aged 74, it pays $375."
+us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Concluded that Social Security-only income produces no refundable credit, ignoring the real property tax credit. That credit depends on property taxes paid and age, not taxable income, and pays this homeowner the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Reported the NY household credit as refundable, but it is a nonrefundable credit that only offsets liability, which is zero here. It also missed the refundable real property tax credit of $375."
+us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used a $75 cap for a homeowner aged 65 or older. $75 is the top of the under-65 maximum table, and the 65+ maximums run from $375 down to $86, so $75 is not an elderly cap at any income level. Here the elderly cap is $375."
+us,scenario_118,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Made up a 2.5%-of-property-taxes formula. The credit is the property taxes above a small percentage of household gross income, capped at the 65+ maximum. With $1,634 in taxes and almost no income, that gives $375."
+us,scenario_118,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Added Social Security and estimated SSI to household gross income, which put the 65+ filer in the $12,001–$13,000 band with a $171 cap. The engine's household income measure for this credit leaves out these nontaxable benefits, so the filer sits in the lowest band and gets the full $375 elderly maximum."
+us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no NY credit applies and never evaluated the real property tax credit. That credit pays this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Treated the NY EITC and child credits as the only refundable credits. It missed the real property tax credit, which pays this low-income 65+ homeowner $375 regardless of earned income."
+us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Wrongly said property tax credits don't apply here. A 74-year-old homeowner paying $1,634 in property taxes with household gross income under $18,000 qualifies for the refundable real property tax credit at the $375 elderly maximum."
+us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Went through EITC, child, tuition, and child care credits but left out the real property tax credit. The household qualifies for that credit through its $1,634 in property taxes and the head's age of 74, worth $375."
+us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Made refundable credits depend on NY tax liability or children. The real property tax credit is paid with zero liability and gives this 65+ homeowner $375.
+us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Only checked the NY EITC and the refundable child and dependent care credit. It missed the real property tax credit, which needs no earnings or dependents and pays this 74-year-old homeowner $375."
+us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Tied refundable credits to taxable income, earned income, or children. The NY real property tax credit depends on property taxes paid, age 65+, and household gross income of $18,000 or less, and pays $375 here."
+us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Wrongly said age and income rule out every NY refundable credit. The head's age of 74 is exactly what makes them eligible for the elderly real property tax credit, which pays the $375 maximum against $1,634 in property taxes."
us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Tested only age under 19 and income of about 197% FPL against the 200% FAMIS limit. It ignored that Child 1 has employer-sponsored insurance, and a child covered under a group health plan is not a CHIP targeted low-income child."
us,scenario_119,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"Correctly placed income (about 197% FPL) under the 200% FAMIS limit plus the 5% disregard. It then stopped there and never applied CHIP's rule excluding children already covered by employer-sponsored insurance, which Child 1 has."
us,scenario_119,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Checked MAGI of $52,569 (about 197% FPL) against the 200% CHIP limit and confirmed the child is not Medicaid-eligible. It skipped the remaining categorical condition: Child 1 is covered by employer-sponsored insurance, which bars CHIP eligibility."
@@ -9346,52 +9577,53 @@ us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,
us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,250 post-TCJA head-of-household deduction, three $5,300 personal exemptions, 15% brackets, and a $2,000 total CTC. OBBBA keeps the $24,150 deduction with no exemptions and a $2,200-per-child CTC. That CTC absorbs the $2,310 left after the $546 CDCC and $200 Saver's credit."
us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It gave no parseable value and no explanation for federal_income_tax_before_refundable_credits.
us,scenario_119,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed as single with a $16,100 standard deduction and single brackets, which gave $5,328 of tax. A parent with two qualifying children files as head of household with a $24,150 deduction, and that tax is $3,056.23. It also used a 20% CDCC instead of 35% and left out the Saver's credit. Under head-of-household status, its correct $4,400 CTC fully offsets the liability."
-us,scenario_119,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It said AGI of $52,569 is above the two-child EITC phase-out and set the EITC to $0. The 2026 phase-out ends near $58,629, and the EITC on $55,000 of earned income is $764.23. For the CTC, it reported the $3,400 per-child refundable cap instead of the unused CTC. Once the $546 CDCC and $200 Saver's Credit are taken first, only $2,310.23 of the $4,400 CTC offsets tax, leaving $2,089.77 refundable."
-us,scenario_119,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It phased out the EITC using AGI ($52,569), which gave $1,268. The phase-out base is the greater of AGI and earned income, $55,000, which gives $764.23. It also left out the $200 Saver's Credit when finding the tax the CTC absorbs: $2,514 instead of $2,310.23. That understated the refundable CTC as $1,886 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It said $55,000 is above the two-child EITC phase-out, but the 2026 phase-out runs until about $58,629, so the EITC is $764.23. It also claimed the refundable CTC has an income cutoff at this level. In fact, the additional CTC is simply the $4,400 CTC left over after $2,310.23 of tax, which is $2,089.77, limited only by the $1,700-per-child cap and 15% of earnings over $2,500."
-us,scenario_119,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used $2,000 per child ($4,000) instead of the 2026 amount of $2,200 ($4,400). It let about $3,100 of tax absorb the CTC without first subtracting the $546 CDCC and $200 Saver's Credit, so it got $900 instead of $2,089.77. It also put the EITC's full phase-out near $52,900, when the 2026 two-child credit is still $764.23 at $55,000 of earned income."
-us,scenario_119,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It estimated taxable income at about $48,300 and tax above $5,000, and concluded the CTC was fully absorbed. With the $24,150 head-of-household standard deduction, taxable income is $28,419 and tax is about $3,056. The $546 CDCC and $200 Saver's Credit cut that to $2,310.23, leaving $2,089.77 of refundable CTC. It also zeroed the EITC, which is $764.23."
-us,scenario_119,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It claimed no EITC, but the two-child credit is $764.23 at $55,000 of earned income. It also assumed only about $800 of CTC was used against tax and applied a $1,800-per-child cap. In fact, $2,310.23 of the $4,400 CTC is absorbed, so the refundable part is $2,089.77, which is below the $1,700-per-child ($3,400) cap for 2026."
-us,scenario_119,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It computed the EITC phase-out on AGI of $52,569, which gave $1,276.20. The phase-out uses the greater of AGI and earned income ($55,000), which gives $764.23. It subtracted the CDCC but not the $200 Saver's Credit (10% of $2,000 in contributions) before the CTC, so it put the absorbed tax at $2,510.28 instead of $2,310.23 and the refundable CTC at $1,889.72 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used outdated parameters: a $2,000 CTC per child instead of $2,200, a $22,500 head-of-household standard deduction instead of $24,150, and a 10% bracket ending at $16,550. It skipped the $546 CDCC and $200 Saver's Credit before the CTC, which gave $723 of refundable CTC instead of $2,089.77. It also ran the EITC on 401(k)-reduced earnings of $51,913 with guessed parameters, getting $922 instead of $764.23 (the 2026 credit on $55,000). It then rounded the total to $1,600."
-us,scenario_119,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It assumed tax fully absorbs the CTC. After the $546 CDCC and $200 Saver's Credit, tax is only $2,310.23, so $2,089.77 of the $4,400 CTC is refundable. It also said the EITC was fully phased out, but the two-child credit is still $764.23 at $55,000 of earned income."
-us,scenario_119,federal_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It computed the EITC on AGI of $52,569 ($1,276) instead of the greater of AGI and earned income, $55,000 ($764.23). It used a 20% CDCC ($312) instead of the 2026 minimum rate of 35% ($546) and left out the $200 Saver's Credit. As a result, too much tax absorbed the CTC, and it got $1,642 of refundable CTC instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It computed the EITC on an AGI of $46,180, which subtracts the $6,389 employer-sponsored insurance premiums, and phased it out from the wrong starting point, getting $3,913. The 2026 two-child credit on $55,000 of earned income, phased out above $23,890, is $764.23. It also used a $4,000 CTC instead of $4,400, so its refundable CTC was $1,597 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used against tax. Only $2,310.23 of the $4,400 CTC offsets tax after the CDCC and Saver's Credit, so $2,089.77 is refundable. It also computed the EITC on AGI of $52,569 with a $7,264 maximum and a $24,871 phase-out start, instead of 2026's $7,316 and $23,890 applied to $55,000 of earned income ($764.23)."
-us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It set the refundable CTC to $0 because it assumed the CTC was fully used. Tax after the CDCC and Saver's Credit is only $2,310.23, so $2,089.77 of the $4,400 CTC is refunded. Its EITC of $1,075.76 was phased out on AGI of $52,569 instead of $55,000 of earned income, which gives $764.23."
-us,scenario_119,federal_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It treated the CTC as a single $2,000 credit that is fully used. For two children, the 2026 CTC is $4,400, and only $2,310.23 of it offsets tax, leaving $2,089.77 refundable. Its EITC used a $7,350 maximum, a $22,800 threshold, and AGI of $52,569. The correct parameters are $7,316 and $23,890, applied to $55,000 of earned income, which gives $764.23."
-us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It based the EITC on an AGI of $46,180, which subtracts the $6,389 employer-sponsored insurance premiums, and got $2,643. The phase-out base is $55,000 of earned income, which gives $764.23. Its refundable CTC of $427 also let far too much tax absorb the credit: only $2,310.23 of the $4,400 CTC is used, so $2,089.77 is refundable."
-us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It reported a flat $3,000 as the refundable CTC and left out the EITC. The refundable CTC is $4,400 minus $2,310.23 of tax (after the $546 CDCC and $200 Saver's Credit), which is $2,089.77. The two-child EITC at $55,000 of earned income adds $764.23."
-us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It overstated the EITC at about $1,933 by applying the phase-out to income well below the $55,000 earned-income base. That base gives $764.23. It understated the refundable CTC at $172 by letting almost all of the CTC offset tax. After the $546 CDCC and $200 Saver's Credit, only $2,310.23 is absorbed, leaving $2,089.77 refundable."
-us,scenario_119,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut earned income to $45,524 by subtracting both the $6,389 employer-sponsored insurance premiums and the $3,087 401(k) deferral, which inflated the EITC to $1,808. On $55,000 of earned income the EITC is $764.23. It also said the CTC was fully used, but $2,089.77 of the $4,400 CTC remains after $2,310.23 of tax."
-us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave $0 with no reasoning. At $55,000 of earned income, the two-child EITC is $764.23, and tax after the CDCC and Saver's Credit ($2,310.23) absorbs only part of the $4,400 CTC. That leaves $2,089.77 of refundable CTC, so its answer drops both credits."
-us,scenario_119,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It gave no breakdown, and its $2,216.39 is $637.62 short of the correct total. The correct total is the $764.23 EITC (on $55,000 of earned income, phased out above $23,890) plus $2,089.77 of refundable CTC. The refundable CTC equals $4,400 minus the $2,310.23 of tax left after the $546 CDCC and $200 Saver's Credit, and the model's total understates that combination."
-us,scenario_119,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no breakdown, and its $1,530.82 is $1,323.19 short of the correct total. The EITC is $764.23 on $55,000 of earned income. The refundable CTC is $2,089.77 because only $2,310.23 of the $4,400 CTC offsets tax after the $546 CDCC and $200 Saver's Credit. The model let far more of the CTC offset tax than that."
-us,scenario_119,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"Its refundable CTC of $1,044 means it let about $3,356 of tax absorb the CTC, rather than subtracting the $546 CDCC and $200 Saver's Credit first. That leaves $2,310.23 absorbed and $2,089.77 refundable. Its EITC of $497 used pre-2026 parameters. The 2026 two-child maximum of $7,316, phased out at 21.06% above $23,890 on $55,000, gives $764.23."
-us,scenario_119,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It said earned income of $51,913 was above the EITC phase-out and set the EITC to $0. The 2026 two-child phase-out ends near $58,629, and the credit on $55,000 is $764.23. It also used a $4,000 CTC instead of $4,400 and let $2,894 of tax absorb it instead of $2,310.23, so it got $1,106 of refundable CTC instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there was no answer to score."
-us,scenario_119,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It treated the full $2,000 per child as refundable. In fact, the CTC first offsets the $2,310.23 of tax left after the CDCC and Saver's Credit, and only the $2,089.77 remainder of the $4,400 CTC is refundable, within the $1,700-per-child cap. It also left out the $764.23 two-child EITC."
-us,scenario_119,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It gave $190 with no derivation, which leaves out almost all of both credits. The correct total is the $764.23 EITC (on $55,000 of earned income) plus $2,089.77 of refundable CTC ($4,400 CTC minus $2,310.23 of tax after the CDCC and Saver's Credit)."
-us,scenario_119,federal_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used against tax. Tax after the $546 CDCC and $200 Saver's Credit is $2,310.23, so $2,089.77 of the $4,400 CTC is refundable. It also phased out the EITC on $52,569, which gave $1,496. The phase-out base is the greater of AGI and earned income ($55,000), which gives $764.23."
-us,scenario_119,federal_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It set the EITC at about $3,000, far above the $764.23 that the 2026 two-child formula gives at $55,000 of earned income. It also set the refundable CTC at $2,363.10, which assumes only $2,036.90 of tax is absorbed. Tax after the CDCC and Saver's Credit is $2,310.23, so the refundable amount is $2,089.77."
-us,scenario_119,federal_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its $1,656 refundable CTC equals $4,400 minus tax after only a 20% CDCC ($312). The 2026 CDCC rate at this AGI is 35% ($546), and the $200 Saver's Credit also comes before the CTC, so the refundable CTC is $2,089.77. Its $1,276 EITC used AGI of $52,569 instead of $55,000 of earned income, which gives $764.23."
-us,scenario_119,federal_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"Its $1,656 refundable CTC comes from subtracting only a 20% CDCC ($312) before the CTC. The 2026 rate at this AGI is 35% ($546), and the $200 Saver's Credit comes first as well. That leaves $2,310.23 of tax absorbing the CTC and $2,089.77 refundable. Its $785 EITC also drifts from the $764.23 given by the $7,316 maximum and $23,890 threshold applied to $55,000."
-us,scenario_119,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It subtracted the $6,389 employer-sponsored insurance premiums from AGI, which gave $46,180. That lowered tax after the CDCC and Saver's Credit to $1,543.60, raised the refundable CTC to $2,856.40, and raised the EITC to $2,621.73. With the correct AGI of $52,569 and earned income of $55,000, tax is $2,310.23, the refundable CTC is $2,089.77, and the EITC is $764.23."
-us,scenario_119,federal_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It let the full $3,056 of regular tax absorb the CTC, which left $1,344. The $546 CDCC and $200 Saver's Credit are applied first, leaving $2,310.23 of tax and $2,089.77 of refundable CTC. Its $1,246 EITC was phased out on AGI instead of $55,000 of earned income, which gives $764.23."
-us,scenario_119,federal_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It computed the refundable CTC as $4,400 minus the full $3,056.28 of regular tax ($1,343.72), leaving out the $546 CDCC and $200 Saver's Credit that come before the CTC. Including them gives $2,089.77. Its EITC of about $1,474 was not phased out on the $55,000 earned-income base, which gives $764.23."
-us,scenario_119,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It ordered the $546 CDCC and $200 Saver's Credit correctly, but it treated the $6,389 employer-sponsored insurance premiums as reducing AGI and earned income to about $46,180. That cut tax to $1,543.60, which inflated the refundable CTC to $2,856.40 and the EITC to $2,623.83. On AGI of $52,569 and earned income of $55,000, the correct amounts are $2,089.77 and $764.23."
-us,scenario_119,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said the household is above the EITC phase-out and has no other refundable credits. The two-child EITC is $764.23 at $55,000 of earned income. The CTC is $4,400, and after $2,310.23 of tax, $2,089.77 of it is refundable."
-us,scenario_119,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It treated the CTC as a single $2,000 credit that is fully used. For two children, the 2026 CTC is $4,400, and only $2,310.23 of it offsets tax, leaving $2,089.77 refundable. Its $1,300 EITC used AGI of $52,569 and estimated parameters instead of $55,000 of earned income with the $7,316 maximum and $23,890 threshold, which give $764.23."
-us,scenario_119,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used the pre-2018 CTC of $1,000 per child and treated it as fully used. The 2026 CTC is $2,200 per child ($4,400), and $2,089.77 of it is refundable beyond the $2,310.23 of tax. It also phased out the EITC on AGI of $52,569 from a $23,200 threshold, getting $1,115. The correct base is $55,000 with a $23,890 threshold, which gives $764.23."
-us,scenario_119,federal_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"Its $806 of unused CTC understates the refundable CTC. After the $546 CDCC and $200 Saver's Credit, only $2,310.23 of the $4,400 CTC is absorbed, so $2,089.77 is refundable. Its $1,301 EITC was phased out on AGI of $52,569 instead of the $55,000 earned-income base, which gives $764.23."
-us,scenario_119,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It swapped the sizes of the two credits. It put the EITC at $4,109, but the 2026 two-child credit at $55,000 of earned income is $764.23. It put the refundable CTC at $584, but with only $2,310.23 of the $4,400 CTC absorbed by tax, $2,089.77 is refundable."
-us,scenario_119,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It phased out the EITC on 401(k)-reduced earnings of $51,913, getting $1,188. The base is $55,000, which gives $764.23. It let $2,742 of tax absorb the CTC by using too small a CDCC and leaving out the $200 Saver's Credit, so its refundable CTC was $1,658 instead of $2,089.77. The two errors partly offset each other, leaving the total $8.01 short."
-us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there was no answer to score."
-us,scenario_119,federal_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It got the EITC right at $764.23 but computed regular tax of $4,128.28 using the single standard deduction ($16,100) and single brackets. As head of household, the deduction is $24,150 and tax is about $3,056. It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit. Together these left only $583.72 of refundable CTC instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It said $55,000 is above the two-child EITC limit, but the 2026 phase-out runs until about $58,629, so the EITC is $764.23. It also said tax exceeds the CTC. After the $546 CDCC and $200 Saver's Credit, tax is $2,310.23, well below the $4,400 CTC, so $2,089.77 is refundable."
-us,scenario_119,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It put the two-child EITC ceiling near $48,000 and zeroed the credit. The 2026 phase-out runs from $23,890 at 21.06% to about $58,629, which gives $764.23 at $55,000. It also claimed tax absorbed the entire $4,400 CTC, but tax after the CDCC and Saver's Credit is $2,310.23, leaving $2,089.77 refundable."
-us,scenario_119,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It filed the single parent as married filing jointly, used AGI of $55,800, and applied a made-up $49,084 phase-out start, which inflated the EITC to about $6,400. As head of household, the phase-out starts at $23,890 and applies to $55,000 of earned income, giving $764.23. It also used a $4,000 CTC instead of $4,400, so its refundable CTC was $1,837 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It denied the EITC because of the $300,000 in stock. The EITC has no asset test; it disqualifies only investment income above about $12,200, and this household has $800 of interest. It qualifies for $764.23. Its $565 refundable CTC also let too much tax absorb the CTC: after the CDCC and Saver's Credit, $2,089.77 of the $4,400 CTC is refundable."
+us,scenario_119,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It set the EITC to $0, but the 2026 two-child EITC phases out at 21.06% above $23,890 and is still $764.23 at $55,000 of earnings. It then set the ACTC at the $3,400 cap ($1,700 × 2) instead of the unused CTC. That is $4,400 minus the $2,310.23 net liability left after the $546 CDCC and $200 Saver's Credit, or $2,089.77."
+us,scenario_119,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed the ACTC as $4,400 − $2,514 and left out the $200 Saver's Credit (10% of $2,000 of retirement contributions at AGI $52,569). That credit lowers the liability the CTC absorbs to $2,310.23, so the ACTC is $2,089.77. Its $1,268 EITC is phased out from AGI $52,569 rather than PolicyEngine's $55,000 earned-income base (the greater of AGI and earned income), which yields $764.23."
+us,scenario_119,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It wrongly said $55,000 is past the EITC phase-out and that the Additional CTC disappears at this income. The two-child EITC is still $764.23 at $55,000. The ACTC has no income ceiling at this level and refunds the $2,089.77 of the $4,400 CTC that exceeds the $2,310.23 net tax liability."
+us,scenario_119,federal_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It said there was no EITC and no CTC left over. Net tax before the CTC is only $2,310.23 after the $546 CDCC and $200 Saver's Credit, so $2,089.77 of the $4,400 CTC is refundable. The two-child EITC is still $764.23 at $55,000 of earnings."
+us,scenario_119,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used the pre-OBBBA $2,000-per-child CTC ($4,000) and a ~$3,100 tax that ignores the $546 CDCC and $200 Saver's Credit. That gave a $900 ACTC instead of $4,400 − $2,310.23 = $2,089.77. It also zeroed the EITC by putting the two-child phase-out end near $52,900; the 2026 credit runs out near $58,629 and is $764.23 at $55,000."
+us,scenario_119,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It claimed ~$48,300 of taxable income and ~$5,000 of tax after a $32,300 deduction, which overstates liability. Under HoH rules, $52,569 − $24,150 = $28,419 of taxable income and $3,056.28 of tax, which the CDCC and Saver's Credit cut to $2,310.23. That leaves $2,089.77 of the $4,400 CTC refundable; the model also used $2,000 per child. It also zeroed the $764.23 EITC."
+us,scenario_119,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It zeroed the EITC, which is $764.23 at $55,000 of earnings with two children. It also capped the ACTC at a $1,800-per-child figure and assumed tax absorbed only ~$800 of the CTC. The CTC actually offsets a $2,310.23 net liability, so the ACTC is $4,400 − $2,310.23 = $2,089.77."
+us,scenario_119,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It left out the $200 Saver's Credit (10% of $2,000 of qualifying retirement contributions at AGI $52,569). Its pre-CTC liability of $2,510.28 was therefore $200 too high, giving an ACTC of $1,889.72 instead of $2,089.77. Its $1,276.20 EITC phases out from AGI $52,569, while PolicyEngine's phase-out base is the $55,000 of wages, which yields $764.23."
+us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used the pre-OBBBA $2,000-per-child CTC and a $22,500 HoH standard deduction, and ignored both the $546 CDCC and the $200 Saver's Credit. That gave a $723 ACTC instead of $2,089.77. It also put the EITC at $922 using guessed parameters on $51,913 of earnings, versus $764.23 on the $55,000 base, then rounded $1,645 down to $1,600."
+us,scenario_119,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It assumed tax liability fully absorbs the CTC and that income is too high for the EITC. Tax before the CTC is only $2,310.23 after the CDCC and Saver's Credit, so $2,089.77 of the $4,400 CTC is refundable. The two-child EITC is $764.23 at $55,000."
+us,scenario_119,federal_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It applied a 20% CDCC rate ($312) instead of the 35% rate OBBBA sets for 2026 at this AGI ($546), and left out the $200 Saver's Credit. Its ACTC was therefore $1,642 rather than $2,089.77. Its $1,276 EITC uses AGI $52,569 as the phase-out base instead of the $55,000 of earnings, which gives $764.23."
+us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut income to $46,180 by subtracting the $6,389 employer-sponsored insurance premiums, though PolicyEngine's AGI is $52,569 and the EITC phase-out base is $55,000. With wrong parameters, that inflated the EITC to $3,913 versus $764.23. It also used a $2,000-per-child CTC and a $2,403 liability, giving an ACTC of $1,597 instead of $2,089.77."
+us,scenario_119,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It said the nonrefundable CTC was fully used, but the $4,400 2026 CTC is set against only $2,310.23 of net tax, leaving a $2,089.77 refundable ACTC. Its $1,431 EITC uses misremembered parameters (max $7,264, start $24,871) on AGI. The 2026 values are $7,316 and $23,890, and on the $55,000 earnings base they give $764.23."
+us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It set the ACTC to $0, claiming the CTC was fully used nonrefundably. A $4,400 CTC against $2,310.23 of net tax leaves $2,089.77 refundable. Its $1,075.76 EITC also exceeds the $764.23 from phasing out at 21.06% above $23,890 on $55,000 of earnings."
+us,scenario_119,federal_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It treated a $2,000 CTC as fully absorbed and set the ACTC to $0. The 2026 credit is $2,200 per child ($4,400), which exceeds the $2,310.23 net liability by $2,089.77. Its EITC used wrong parameters (max $7,350, start $22,800) on AGI $52,569, giving $1,080.65 instead of $764.23."
+us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 employer insurance premiums, which inflated the EITC to $2,643 versus $764.23 on the $55,000 earnings base. It also used a $2,000 per-child CTC and refunded only $427. The correct ACTC is $4,400 minus $2,310.23 of net tax, or $2,089.77."
+us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It stated a flat $3,000 refundable CTC without computing tax liability, and left out the EITC entirely. The unused CTC is $4,400 − $2,310.23 = $2,089.77, and the EITC adds $764.23."
+us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It refunded only $172 of CTC, so it treated far more of the credit as absorbed than the $2,310.23 net liability. That liability comes from the $24,150 HoH deduction, the $546 CDCC and the $200 Saver's Credit, and it leaves a $2,089.77 ACTC. Its $1,933 EITC is too high; the 21.06% phase-out on $55,000 gives $764.23."
+us,scenario_119,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It subtracted both the $3,087 401(k) deferral and the $6,389 employer insurance premiums to get $45,524 of earned income. That put the EITC at $1,808 versus $764.23 on the $55,000 phase-out base. It also set the ACTC to $0, though $2,089.77 of the $4,400 CTC exceeds the $2,310.23 net liability."
+us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned $0 with no computation. That misses both the $2,089.77 ACTC (the CTC left over after a $2,310.23 net liability) and the $764.23 EITC still available to a two-child HoH filer at $55,000 of earnings."
+us,scenario_119,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It gave $2,216.39 with no breakdown, $637.62 short of the correct total. That total is the $764.23 EITC plus a $2,089.77 ACTC, where the ACTC is the $4,400 CTC minus $2,310.23 of net liability after the $546 CDCC and $200 Saver's Credit."
+us,scenario_119,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"Its $1,530.82 is below the $2,089.77 ACTC alone, so it understated the unused CTC. That is $4,400 minus a net liability of only $2,310.23 after the $546 CDCC and $200 Saver's Credit, and the $764.23 EITC comes on top of it."
+us,scenario_119,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"Its $1,044 ACTC implies a pre-CTC liability well above the actual $2,310.23 (tax of $3,056.28 less the $546 CDCC and $200 Saver's Credit); the correct ACTC is $2,089.77. Its $497 EITC is below the $764.23 that results from phasing out $7,316 at 21.06% above $23,890 on $55,000."
+us,scenario_119,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It used a $4,000 CTC ($2,000 per child) instead of the 2026 $2,200-per-child $4,400, and overstated nonrefundable use at $2,894 versus $2,310.23. That gave a $1,106 ACTC instead of $2,089.77. It also zeroed the EITC, claiming $51,913 is past the phase-out, but the two-child 2026 phase-out ends near $58,629 and the EITC is $764.23."
+us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no answer for federal_refundable_credits. The correct value is the $764.23 EITC plus the $2,089.77 ACTC, or $2,854.01."
+us,scenario_119,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It treated the full $2,000 per child as refundable. That ignores that the CTC first offsets the $2,310.23 net liability, that the 2026 credit is $2,200 per child, and that the refundable part is capped at $1,700 per child. The ACTC is $2,089.77, and the model also left out the $764.23 EITC."
+us,scenario_119,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It gave $190 with no derivation, missing nearly all of the $2,089.77 ACTC ($4,400 CTC minus $2,310.23 of net tax) and the $764.23 EITC."
+us,scenario_119,federal_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used nonrefundably, but net tax after the $546 CDCC and $200 Saver's Credit is only $2,310.23 against a $4,400 CTC, so $2,089.77 is refundable. Its $1,496 EITC also exceeds the $764.23 obtained on the $55,000 phase-out base."
+us,scenario_119,federal_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It put the EITC at about $3,000 instead of $764.23 (the $7,316 maximum less 21.06% of $55,000 − $23,890). It also overstated the ACTC at $2,363.10 versus $2,089.77 by understating the $2,310.23 net liability the CTC offsets."
+us,scenario_119,federal_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its $1,656 ACTC reflects a 20% CDCC ($312) instead of the 2026 35% rate ($546) and no $200 Saver's Credit; the correct ACTC is $2,089.77. Its $1,276 EITC phases out from AGI $52,569 rather than the $55,000 earnings base, which gives $764.23."
+us,scenario_119,federal_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"Its $1,656 ACTC uses a $312 CDCC (20%) instead of $546 (35%) and leaves out the $200 Saver's Credit. That understates the ACTC by about $434 against $2,089.77, while its $785 EITC is close to $764.23."
+us,scenario_119,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It subtracted the $6,389 employer insurance premiums to reach an AGI of $46,180. That understated tax, so only $1,543.60 of CTC was used instead of $2,310.23 and the ACTC rose to $2,856.40 instead of $2,089.77. It also produced a $2,621.73 EITC instead of $764.23 on the $55,000 phase-out base."
+us,scenario_119,federal_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,344 ACTC is $4,400 minus the full $3,056 regular tax. It left out the $546 CDCC and $200 Saver's Credit, which cut liability to $2,310.23 and raise the ACTC to $2,089.77. Its $1,246 EITC also exceeds the $764.23 from the 21.06% phase-out on $55,000."
+us,scenario_119,federal_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"Its ACTC of $1,343.72 = $4,400 − $3,056.28 ignores the $546 CDCC and $200 Saver's Credit applied before the CTC, so the ACTC should be $2,089.77. Its ~$1,474 EITC is above the $764.23 from the 21.06% phase-out on $55,000."
+us,scenario_119,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It applied the $546 CDCC and $200 Saver's Credit, but subtracted the $6,389 employer insurance premiums to reach an AGI of $46,180. That understated tax at $2,289.60, so the ACTC became $2,856.40 instead of $2,089.77. The EITC came out $2,623.83 instead of $764.23 on the $55,000 phase-out base."
+us,scenario_119,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said income is past the EITC phase-out and missed the ACTC entirely. At $55,000 the two-child EITC is still $764.23, and the $4,400 CTC exceeds the $2,310.23 net liability by $2,089.77, all of it refundable."
+us,scenario_119,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used a $2,000 total CTC, entirely absorbed as nonrefundable. The 2026 CTC is $2,200 per child ($4,400) against $2,310.23 of net tax, leaving a $2,089.77 ACTC. Its $1,300 EITC phases out from AGI $52,569 instead of the $55,000 earnings base, which gives $764.23."
+us,scenario_119,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used a $1,000-per-child CTC instead of $2,200 and so found no ACTC, whereas $4,400 minus $2,310.23 of net liability leaves $2,089.77 refundable. Its $1,115 EITC uses AGI $52,569 and a $23,200 start, versus the $23,890 start on the $55,000 earnings base, which gives $764.23."
+us,scenario_119,federal_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"Its $806 ACTC implies a pre-CTC liability far above the $2,310.23 net tax (regular tax of $3,056.28 less the $546 CDCC and $200 Saver's Credit); the ACTC is $2,089.77. Its $1,301 EITC phases out from AGI $52,569 instead of $55,000, which gives $764.23."
+us,scenario_119,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its $4,109 EITC skips most of the 21.06% phase-out from $23,890 to $55,000, which leaves $764.23. Its $584 ACTC overstates how much CTC tax absorbs: only $2,310.23, leaving $2,089.77 refundable."
+us,scenario_119,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"Its errors nearly cancelled. The $1,658 ACTC used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit, understating the ACTC by about $432 against $2,089.77. Its $1,188 EITC phases out on $51,913 rather than the $55,000 earnings base, which yields $764.23."
+us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer for federal_refundable_credits. The correct value is the $764.23 EITC plus the $2,089.77 ACTC, or $2,854.01."
+us,scenario_119,federal_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It computed tax as a single filer ($16,100 deduction and a $12,400 10% bracket, giving $4,128.28) instead of head of household ($24,150 and $17,700, giving $3,056.28). It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit. That made the ACTC $583.72 instead of $2,089.77; its $764.23 EITC was right."
+us,scenario_119,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It said $55,000 exceeds the EITC limits and that tax exceeds the CTC. The two-child EITC phases out completely only near $58,629 and is $764.23 at $55,000. Net tax of $2,310.23 is well below the $4,400 CTC, leaving $2,089.77 refundable."
+us,scenario_119,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It put the two-child EITC ceiling near $48,000, but the 2026 phase-out ends near $58,629, which gives $764.23 at $55,000. It also said the entire $4,400 CTC was absorbed, though net tax after the CDCC and Saver's Credit is $2,310.23, leaving a $2,089.77 ACTC."
+us,scenario_119,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the single parent as married filing jointly and used an invented ~$49,084 phase-out start on $55,800 of income. That produced a $6,400 EITC instead of $764.23 under head-of-household rules. It also used a $4,000 CTC, giving an ACTC of $1,837 instead of $2,089.77."
+us,scenario_119,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It denied the EITC because of the $300,000 in stock, but the EITC has no asset test, only an investment-income limit, and $800 of interest is far below it. The EITC is $764.23. Its $565 ACTC also understates the $2,089.77 left after the $4,400 CTC offsets $2,310.23 of net tax."
us,scenario_119,free_school_meals_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,head_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -9425,52 +9657,53 @@ us,scenario_119,self_employment_tax,glm-5.3,parse_contract_failure,missing_outpu
us,scenario_119,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,snap,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It correctly reached Virginia taxable income of $39,469 (AGI less the $8,750 standard deduction, $2,790 of exemptions and the $1,560 child and dependent care deduction) and bracket tax of $2,012. It then dropped that result for an unsupported $2,296 that matches no bracket computation. It also never subtracted Virginia's 20% nonrefundable EITC ($152.85), which reduces tax before refundable credits."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed taxable income ($39,469) and bracket tax ($2,012) correctly but treated the Virginia EITC as refundable only. Here Virginia's 20%-of-federal EITC is taken as a nonrefundable credit ($152.85 on the $764.23 federal EITC), which lowers tax before refundable credits to $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It used the obsolete $4,500 single standard deduction instead of the 2026 $8,750 amount and left out both the three $930 personal exemptions and the $1,560 child and dependent care deduction, so taxable income came out at $48,069 instead of $39,468.54. Its $1,888 does not match bracket tax on its own $48,069 ($2,506), and it applied no nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of the 2026 $8,750 and left out the $1,560 child and dependent care expense deduction, which gave $41,279 of taxable income and $2,116 of tax instead of $39,468.54 and $2,011.94. It then subtracted an unquantified ~$145 'childcare credit' (Virginia has no childcare credit, only the deduction) instead of the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care deduction, reaching $2,116, then raised that to $2,459 with no stated basis. It applied no nonrefundable Virginia EITC, which takes $152.85 off the correct $2,011.94 bracket tax."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and skipped the $1,560 child and dependent care deduction (taxable income $41,279, tax $2,116). It then added unexplained upward 'adjustments' to reach $2,280 instead of subtracting the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its Virginia taxable income ($39,469) and bracket tax ($2,011.97) match the derivation, so the whole gap is the Virginia EITC. It set that credit at 20% of a $1,276.20 federal EITC phased out on $52,569 of AGI, while the reference's $764.23 federal EITC is phased out on $55,000 of gross wages before the $3,087 traditional 401(k) deferral. That makes the reference's Virginia credit $152.85 instead of the model's $255.24."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used a $9,000 standard deduction (the 2026 Virginia amount is $8,750) and never applied the $1,560 child and dependent care expense deduction, which gave $40,779 of taxable income and $2,087 of tax. It also said no nonrefundable Virginia credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It subtracted only the $8,750 standard deduction and left out the three $930 exemptions and the $1,560 child and dependent care deduction, so taxable income was $43,819 instead of $39,468.54. It then subtracted an invented $220 low-income adjustment and a federal-style 30% childcare credit instead of the $152.85 nonrefundable EITC. Virginia gives childcare relief only as a deduction, and its low-income credit requires income at or below the poverty line."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It applied the $8,750 standard deduction and $2,790 of exemptions but left out the $1,560 child and dependent care expense deduction (taxable income $41,029 instead of $39,468.54; tax $2,102 instead of $2,011.94). It subtracted no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by also subtracting the $6,389 employer-sponsored insurance premiums, but AGI is $52,568.54 (wages plus interest, less only the $3,087 traditional 401(k) and the $144 IRA). It then used a ~$9,470 standard deduction without the exemptions or the $1,560 childcare deduction. Finally it subtracted a $1,000 'Virginia child tax credit' that does not exist instead of the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out the $2,790 of personal exemptions and the $1,560 childcare deduction (taxable income $44,069 instead of $39,468.54). It then subtracted a nonexistent Virginia CDCC set at 40% of the federal credit ($124.80) instead of the 20% nonrefundable Virginia EITC of $152.85."
-us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116.04). It applied no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It used the obsolete $4,500 standard deduction instead of $8,750 and left out the $1,560 childcare deduction. It also misapplied the brackets by stretching the 5% band to $22,000 instead of stopping it at $17,000, charging $850 instead of $600 on that band. It applied no nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It understated taxable income at about $34,780 instead of $39,468.54. It subtracted a $78 Virginia child care credit that does not exist (Virginia gives a deduction). It took the Virginia EITC as 25% of an overstated federal EITC ($660) instead of 20% of the $764.23 federal EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It started from $51,713 of AGI instead of $52,568.54. By its own explanation it subtracted only the standard deduction and exemptions, with no $1,560 child and dependent care deduction. Its $2,043 is bracket tax on about $40,010 of taxable income with no $152.85 nonrefundable Virginia EITC subtracted."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 ESI premiums. It applied a $3,000 standard deduction on the false premise that Virginia's deduction reverts after 2025, when the 2026 amount is $8,750. The lower AGI also inflated its federal EITC, so its 20% Virginia EITC came to $386.60 instead of $152.85."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 ESI premiums. It used an $8,500 standard deduction and outdated $800 exemptions ($2,400) instead of $8,750 and 3 × $930, and left out the $1,560 childcare deduction. It then subtracted a nonexistent Virginia CDCC ($62.40) instead of the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no derivation. The correct path is $52,568.54 of AGI less $8,750, $2,790 and $1,560, giving $39,468.54 of taxable income and $2,011.94 of bracket tax, minus the $152.85 nonrefundable Virginia EITC, for $1,859.09. Its $1,974 is bracket tax on about $38,810 of taxable income with nothing subtracted for the Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It claimed a nonrefundable Virginia child care credit, but Virginia gives childcare relief only as the $1,560 child and dependent care expense deduction. The only nonrefundable credit here is the 20% Virginia EITC ($152.85), and its $1,696.28 is $162.81 below the correct $2,011.94 − $152.85."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no derivation. Its $1,450.48 is bracket tax on about $29,700 of taxable income, roughly $9,800 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560). It therefore over-deducted from income and understated tax by $408.61 compared with $2,011.94 less the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (tax $2,145 instead of $2,011.94). It then subtracted a nonexistent $78 Virginia CDCC and a $100 Virginia EITC instead of the $152.85 nonrefundable EITC (20% of the $764.23 federal EITC)."
-us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It estimated the federal EITC at $5,669.56 without applying the 21.06% phase-out above $23,890, so its 20% Virginia EITC came to $1,133.91 instead of $152.85 (20% of $764.23). It also left out the $2,790 of exemptions and the $1,560 childcare deduction and used $8,500 instead of $8,750, which overstated bracket tax at $2,276.47 against $2,011.94."
-us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,859.09 target."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"Its $803.75 is Virginia bracket tax on only about $18,460 of taxable income, roughly $21,000 below the correct $39,468.54 (AGI of $52,568.54 less the $8,750 standard deduction, $2,790 of exemptions and the $1,560 childcare deduction). It applied no nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It reported $180 after 'nonrefundable credits', which means taking about $1,832 of credits off the $2,011.94 bracket tax. The only nonrefundable Virginia credit this household gets is the 20% EITC of $152.85, so it applied roughly $1,680 of nonexistent credits."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It subtracted only about $7,350 from AGI (taxable income $45,219), although the $8,750 standard deduction, $2,790 of exemptions and $1,560 childcare deduction total $13,100 (taxable income $39,468.54). That overstated bracket tax at $2,343 against $2,011.94. Its $267 Virginia EITC is also above the $152.85 that comes from the $764.23 federal EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,483.02 is bracket tax on about $30,270 of taxable income, roughly $9,200 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560), so it over-reduced income. It also listed no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"It got taxable income and bracket tax ($2,012) right but said no nonrefundable low-income credit is available and applied no credit. That misses Virginia's 20% nonrefundable EITC ($152.85 on the $764.23 federal EITC), which lowers tax before refundable credits to $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child and dependent care deduction (taxable income $41,029, tax $2,101.67). It then subtracted two $500 nonrefundable child credits that Virginia does not offer, instead of the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It started from $46,180 of AGI by also subtracting the $6,389 employer-sponsored insurance premiums, but AGI is $52,568.54 (only the $3,087 traditional 401(k) and $144 IRA come out). That put taxable income at $33,080 instead of $39,468.54. It applied no nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child and dependent care expense deduction (bracket tax $2,101.67 instead of $2,011.94). It then subtracted a ~$63 Virginia dependent-care credit that does not exist, instead of the 20% nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It stopped at the standard deduction and exemptions (taxable income $41,029), leaving out the $1,560 child and dependent care expense deduction. It subtracted no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"Its $33,080 of taxable income comes from $46,180 of AGI, which means it subtracted the $6,389 ESI premiums; the correct figures are $52,568.54 of AGI and $39,468.54 of taxable income. It also misapplied the brackets, reporting $1,764.60 when its own $33,080 yields $1,644.60, and it applied no nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It listed only the standard deduction and exemptions and no Virginia EITC. Its $1,874 is bracket tax on about $37,070 of taxable income, so it over-deducted by about $2,400 against the correct $39,468.54 while also leaving out the $152.85 nonrefundable EITC; the two errors nearly cancel."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116 instead of $2,011.94). It subtracted no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (tax $2,116 instead of $2,011.94). It then took a $223 Virginia EITC instead of $152.85 (20% of the $764.23 federal EITC)."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It applied the childcare deduction and exemptions but used an $8,500 standard deduction instead of $8,750, giving $2,026 of tax against $2,011.94. Most of its gap is the Virginia EITC, which it put at $260 from a federal EITC phased out on AGI. The reference's $764.23 federal EITC is phased out on $55,000 of gross wages, which gives $152.85."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its $35,330 of taxable income is $4,139 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560), which understated bracket tax. It subtracted no nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (taxable income $41,779, tax $2,145 instead of $2,011.94). It then subtracted a $238 Virginia EITC instead of $152.85 (20% of the $764.23 federal EITC)."
-us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,859.09 target."
-us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116.04). It said no nonrefundable state credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It used a $5,200 standard deduction and invented brackets: 3% up to $17,000 and 5% above, instead of 2%/3%/5% up to $17,000 and 5.75% above. It counted four $800 exemptions for a three-person unit and then reported a net tax ($2,867) higher than its own gross tax ($2,120). It left out the $1,560 childcare deduction and the nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116). It said no nonrefundable credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)."
-us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It built a federal AGI of $59,111 by adding imputed employer health-coverage value and never subtracting the $3,087 traditional 401(k) or the $144 IRA (AGI is $52,568.54). It cited an $18,000 joint standard deduction for a single-parent filer. Its reported $1,173.89 is not bracket tax on its own $41,111 (about $2,106), and it left out the childcare deduction and the Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It reported $50,200 of taxable income, subtracting only about $2,370 instead of $13,100 ($8,750 + $2,790 + $1,560). It then claimed a $112 credit for taxable income under $50,250 that Virginia does not have (its low-income credit requires income at or below the poverty line) instead of the $152.85 nonrefundable EITC. Its $2,024 base tax also does not match bracket tax on $50,200 ($2,629)."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It correctly worked taxable income down to $39,469 and tax to $2,012 using the $1,560 child-care subtraction. It then dropped that result for an unexplained $2,296. It also never subtracted the 20% nonrefundable Virginia EITC ($152.85)."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"Its taxable income ($39,469) and pre-credit tax ($2,012) are right. It then treated the Virginia EITC as refundable only and never subtracted the 20% nonrefundable Virginia EITC ($152.85), which this output nets out."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It used the outdated $4,500 single standard deduction instead of $8,750, and left out both the $2,790 personal exemptions and the $1,560 child-care subtraction. It also applied no Virginia EITC. Even its own taxable income of $48,069 gives about $2,506 under Virginia's brackets, not $1,888."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It said Virginia tax was zero. After the $8,750 standard deduction, $2,790 exemptions and $1,560 child-care subtraction, taxable income is still $39,469, which produces $2,011.94 of tax. Only the $152.85 nonrefundable EITC reduces that."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of the 2026 figure of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then took off an unexplained 'small credit' of about $145 instead of the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then raised the figure to $2,459 with no basis and never applied the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then raised the figure to $2,280 for unexplained 'adjustments' instead of subtracting the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its taxable income ($39,469) and pre-credit tax ($2,011.97) match the reference. The whole gap comes from the federal EITC used for the 20% Virginia credit. The model phased the EITC out on AGI of $52,569, getting $1,276.20 and a $255.24 Virginia credit. The reference uses a federal EITC of $764.23 and a $152.85 Virginia credit, which corresponds to phasing out on $55,000 of gross wages."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used a $9,000 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave taxable income of $40,779 and tax of $2,087. It also said no nonrefundable credit applied, missing the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It left out the $2,790 personal exemptions and the $1,560 child-care subtraction, taxing $43,819 to get $2,262.60. It then subtracted an invented $220 low-income credit and a Virginia child-care credit that does not exist, since Virginia gives child care as a deduction. It also skipped the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"The $8,750 standard deduction and $2,790 exemptions are right. It left out the $1,560 child-care subtraction, so taxable income came out at $41,029 instead of $39,469. It also never subtracted the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting $6,389 of ESI premiums from the stated wages, used a made-up $9,470 standard deduction, and left out the exemptions and child-care subtraction. It then subtracted a $1,000 Virginia child tax credit, which is not part of Virginia's 2026 credits, instead of the 20% EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out both the $2,790 personal exemptions and the $1,560 child-care subtraction, taxing $44,069 instead of $39,469. It then applied a '40% of federal' Virginia child-care credit that does not exist, instead of the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279 instead of $39,469). It also never subtracted the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It used the outdated $4,500 standard deduction and left out the $1,560 child-care subtraction. It also got the brackets wrong, applying 5% to $17,000 and 5.75% only above $22,000; the correct brackets put 5% on $5,000–$17,000 and 5.75% on everything above. It applied no Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It understated taxable income at about $34,780 instead of $39,469 and subtracted a Virginia child-care credit that does not exist. It also took the nonrefundable EITC at 25% of an inflated federal EITC ($660), when Virginia's nonrefundable rate is 20% of a $764.23 federal EITC ($152.85)."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It started from an AGI of $51,713 instead of $52,569, and its $2,043 is above the correct pre-credit tax of $2,011.94. That means it left out the $1,560 child-care subtraction and never applied the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums and used a $3,000 standard deduction instead of $8,750; the two errors partly cancel. The lower AGI also inflated the federal EITC, so its 20% Virginia credit came out at $386.60 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums, used $8,500 instead of $8,750, used $800 exemptions instead of $930, and left out the $1,560 child-care subtraction. It then subtracted a Virginia child-care credit that does not exist instead of the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no derivation. The correct steps give $39,469 of taxable income and $2,011.94 of tax, less a $152.85 nonrefundable Virginia EITC. Its $1,974 is close to the pre-credit tax, which shows it skipped most of the EITC offset."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It cited a 'nonrefundable state child care credit' that Virginia does not have, since Virginia allows a $1,560 deduction for these expenses instead. It never applied the 20% nonrefundable EITC. Its $1,696.28 implies an understated taxable base compared with $39,469 taxed at $2,011.94."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no derivation. Its $1,450.48 is $409 below the result of taxing $39,469 ($2,011.94) and subtracting the $152.85 EITC, which implies it understated taxable income or overstated nonrefundable credits."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (tax $2,145 instead of $2,011.94). It then subtracted a Virginia child-care credit that does not exist and a $100 EITC instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It left out the $2,790 exemptions and the $1,560 child-care subtraction and used $8,500 for the standard deduction. It also set the federal EITC at a near-maximum $5,669.56, ignoring the 21.06% phase-out above about $23,890, so its 20% Virginia credit was $1,133.91 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is no answer to score."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It said it subtracted only the standard deduction, yet its $803.75 is less than half the correct figure. The correct steps give taxable income of $39,469, tax of $2,011.94, and $1,859.09 after the 20% EITC, so the model applied the rates to a far too small base."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It reported $180 of residual tax with no derivation. Virginia taxable income after the $8,750 standard deduction, $2,790 exemptions and $1,560 child-care subtraction is $39,469, giving $2,011.94 of tax. Only the $152.85 nonrefundable EITC offsets that."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"Its taxable income of $45,219 subtracts only about $7,350 from AGI. The standard deduction ($8,750), exemptions ($2,790) and child-care subtraction ($1,560) total $13,100, which gives $39,469. It also overstated the nonrefundable Virginia EITC at $267 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,483.02 implies taxable income of about $30,270, not the correct $39,469 (AGI $52,569 less $8,750, $2,790 and $1,560). It also described no 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its deductions and its $2,012 pre-credit tax are right. It said no nonrefundable credit was available and so never subtracted the 20% nonrefundable Virginia EITC ($152.85)."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child-care subtraction (tax $2,101.67 on $41,029 of taxable income). It then subtracted two $500 'child credits' that Virginia does not have, instead of the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 of ESI premiums from the stated gross wages. The correct AGI is $52,568.54 (wages plus interest less the 401(k) and IRA contributions), so its taxable income of $33,080 was $6,389 too low. It also never applied the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It subtracted only the standard deduction and exemptions, leaving out the $1,560 child-care subtraction. It then took off a 'Virginia dependent-care credit' that does not exist instead of the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child-care subtraction (taxable income $41,029 instead of $39,469). It also never subtracted the 20% nonrefundable Virginia EITC ($152.85)."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums, which produced taxable income of $33,080. Virginia's brackets give $1,644.60 on that amount, so its $1,764.60 is also off by $120 in arithmetic. It never applied the 20% nonrefundable EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"Its reasoning mentions only the standard deduction and exemptions, which alone give $2,101.67. What brings the correct answer to $1,859.09 is the $1,560 child-care subtraction and the 20% nonrefundable EITC, and the model applied neither."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279). It also never subtracted the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out the $1,560 child-care subtraction, which gave a tentative tax of $2,116 instead of $2,011.94. It then took a $223 Virginia EITC instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It did include the child-care subtraction and the exemptions, but used an $8,500 standard deduction instead of the 2026 figure of $8,750. That gave taxable income of $39,719 and tax of $2,026. It also subtracted a $260 nonrefundable Virginia EITC instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its taxable income of $35,330 is $4,139 below the correct $39,469 (AGI $52,569 less $8,750, $2,790 and $1,560). It also never applied the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"Its taxable income of $41,779 reflects an $8,000 standard deduction instead of $8,750 and leaves out the $1,560 child-care subtraction. It also took the nonrefundable Virginia EITC at about $238 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is no answer to score."
+us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction. It also said no nonrefundable credit applied, missing the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It used a $5,200 standard deduction and mis-stated the brackets as 3% up to $17,000 and 5% above; Virginia actually taxes 5% on $5,000–$17,000 and 5.75% above that. It counted four $800 exemptions instead of three at $930. It then raised the tax after the exemptions instead of lowering it."
+us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279 instead of $39,469). It also said no nonrefundable credits applied, missing the 20% nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It added imputed employer health coverage to AGI ($59,111) instead of using $52,569, and mixed up the standard deduction with joint-filer and exemption figures. Its $1,173.89 also does not follow from its own taxable income of $41,111, which gives about $2,149 under Virginia's brackets."
+us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"Its taxable income of $50,200 leaves out most of the $8,750 standard deduction and all of the $1,560 child-care subtraction, and its $2,024 base tax does not match Virginia's brackets on that amount. It then subtracted a $112 low-income credit this household does not qualify for, instead of the 20% nonrefundable EITC."
us,scenario_119,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Put 15% of an assumed ~$1,268 federal EITC into refundable credits. It ignored Virginia's rule that filers take either the refundable or the nonrefundable EITC. With about $2,000 of Virginia liability absorbing the credit, the EITC is claimed as nonrefundable and refundable credits are $0."
us,scenario_119,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Treated Virginia's EITC as a refundable 15% of the ~$1,276 federal EITC. Virginia requires a choice between refundable and nonrefundable. This household's roughly $2,000 state liability fully absorbs the credit, so it goes to the nonrefundable credits and state refundable credits are $0."
us,scenario_119,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Booked 20% of a federal EITC of ~$3,913 as a refundable Virginia credit. That federal figure is far too high: with AGI of ~$52.6k, the phaseout leaves about $1.3k. The model also missed that Virginia's EITC is taken in its nonrefundable form when state liability covers it, which leaves $0 of refundable state credits."
@@ -9487,52 +9720,53 @@ us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,Fal
us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_refundable_credits. The correct answer is $0 because Virginia's EITC is taken as a nonrefundable credit against the household's state liability.
us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Put 20% of an understated $764.23 federal EITC into refundable credits. The federal EITC is about $1.3k at this AGI. The decisive error is treating Virginia's EITC as refundable: the household's ~$2,000 Virginia tax absorbs the nonrefundable form, so refundable state credits are $0."
us,scenario_119,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"At the 2026 single brackets, its own ~$200,500 of taxable income produces about $40,700 of tax, so reporting $30,238 misapplies the rate schedule by roughly $10,500. It also capped SALT at the old $10,000 instead of deducting the ~$13,908 allowed under the OBBBA's $40,400 cap. It deducted the full $3,744 of mortgage interest instead of the $2,808 allowed on $750k of $1M debt, and it omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI ($234,414 instead of $232,635), which raised taxable income by about $1,900 and regular tax by about $457. It also computed NIIT on $1,853 after netting the farm rent loss, instead of on the $4,300 of interest and rental income ($163.40). That offsets only $93 of the overstatement."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It took a $30,000 standard deduction instead of itemizing $35,717: $19,001 of charitable gifts after the 0.5% floor, ~$13,908 of SALT including Connecticut income tax, and $2,808 of limited mortgage interest. It counted all $41,900 of Social Security instead of the $35,615 taxable portion. It then subtracted a fabricated $11,297 of withholding, which is a tax payment, not a nonrefundable credit."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket computation gave $40,823 on $200,477 of taxable income, but it reported $38,740 with no credit to justify the $2,083 cut. Its taxable income used the old $10,000 SALT cap instead of the ~$13,908 allowed under the OBBBA's $40,400 cap. It also deducted the full $3,744 of mortgage interest instead of $2,808 under the $750k debt limit, skipped the 0.5%-of-AGI charitable floor, and omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap, skipped the 2026 0.5%-of-AGI charitable floor, never subtracted the $617 401(k) deferral from wages, and added the $1,750 refund. Together these put taxable income at $202,030 instead of $196,918. It also ended the 22% bracket at $107,400 instead of $105,700 and omitted the $163.40 NIIT on $4,300 of interest and rental income."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left Connecticut income tax (~$11,896) out of SALT, deducted the full $3,744 of mortgage interest instead of $2,808 under the $750k/$1M acquisition-debt ratio, and skipped the 0.5% charitable floor, reaching $208,465 of taxable income. At 2026 brackets that income yields about $43,165, so its $40,650 also misapplies the rate schedule. It omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 state tax refund in AGI ($234,385 instead of $232,635), which pushed taxable income to $198,906 and regular tax up by about $477. It also computed NIIT at $70 on $1,853 netted for the farm rent loss, instead of 3.8% of the $4,300 of interest and rental income ($163.40)."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025 and applied pre-TCJA 10/15/25/28/33% brackets, a personal exemption, no SALT cap, and a 2%-floor miscellaneous deduction. The OBBBA made the TCJA rules permanent: the 2026 single schedule is 10/12/22/24%, with no personal exemption and no miscellaneous itemized deductions. It also left Connecticut income tax out of SALT, and its reported $42,046 matches none of its own totals ($49,719, $48,053, $47,592, $47,383)."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $207,651 of taxable income but reported ~$24,500 of tax, while 2026 single brackets give about $42,900 on that amount, a gross misapplication of the rate schedule. It also left the ~$11,896 of Connecticut income tax out of SALT, although it is deductible under the OBBBA's $40,400 cap. It skipped the 0.5% charitable floor and omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $617 traditional 401(k) deferral and added the $1,750 state refund, so AGI was $235,031 instead of $232,635 and regular tax was about $656 too high. It then computed NIIT on $1,853 after netting the farm rent loss, instead of on the $4,300 of interest and rental income ($163.40)."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 employer-sponsored insurance premium from wages, but only the $617 traditional 401(k) deferral reduces the stated gross wages. It also understated SALT at $11,815 and skipped the 0.5% charitable floor. Its own $192,198 of taxable income yields about $38,725 at 2026 brackets, so its $34,322 also misapplies the rates, and it omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as a TCJA-sunset year, with a $5,063 personal exemption, a $3,114 2%-floor miscellaneous deduction, and pre-TCJA brackets. The OBBBA made the TCJA brackets permanent and eliminated personal exemptions and miscellaneous itemized deductions. It also subtracted the $7,429 ESI premium from wages and computed NIIT on $1,853 instead of $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,250 personal exemption, pre-TCJA brackets, and $42,059 of itemized deductions against the correct $35,717. The OBBBA permanently kept the 10/12/22/24% schedule with no personal exemption. It also computed NIIT at $70.41 on investment income netted for the farm rent loss, instead of $163.40 on $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It omitted the $163.40 NIIT on $4,300 of interest and rental income, even though MAGI exceeds the $200,000 threshold. It also added the $1,750 state refund to AGI and deducted charitable gifts without the 2026 0.5%-of-AGI floor, leaving taxable income $1,639 too high at $198,557."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset: no SALT cap, $2,966 of 2%-floor miscellaneous deductions for unreimbursed employee expenses, and pre-TCJA rates. The OBBBA made the TCJA brackets permanent and permanently eliminated miscellaneous itemized deductions. Its $192,187 of taxable income yields about $38,720 at the actual 2026 schedule, so the pre-TCJA rates drove the overstatement; it also omitted NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $39,591 corresponds to about $195,800 of taxable income with no NIIT, which means it overstated deductions by roughly $1,100 against the correct $35,717 itemized total. It also omitted the $163.40 NIIT on $4,300 of interest and rental income."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $7,429 ESI premium, when only the $617 401(k) deferral reduces wages. It itemized 2%-floor miscellaneous employee business expenses, which the OBBBA permanently eliminated, and applied sunset-era rates that produced $41,404 of regular tax on an AGI below the correct $232,635. It computed NIIT at $87 instead of 3.8% of $4,300 ($163.40)."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,000 personal exemption, $3,149 of 2%-floor miscellaneous deductions, and pre-TCJA brackets. The OBBBA made the TCJA schedule permanent and removed those deductions. It also reduced wages by the $7,429 ESI premium and computed NIIT on $1,853 instead of $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. At 2026 single brackets, $33,814 implies about $171,700 of taxable income, roughly $25,200 below the correct $196,918, so the model understated income or overstated deductions by that amount. It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, when only the $617 401(k) deferral reduces gross wages. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap and skipped the 0.5% charitable floor, cutting taxable income to $194,013. It omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $37,824 corresponds to about $188,400 of taxable income, roughly $8,500 below the correct $196,918, which matches subtracting the $7,429 ESI premium from wages as its sibling models did. It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $33,024 implies about $168,400 of taxable income, roughly $28,500 below the correct $196,918, so it understated taxable income by that amount. It also omitted the $163.40 NIIT on $4,300 of interest and rental income."
-us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left Connecticut income tax (~$11,896) out of SALT and deducted only the $2,012 property tax, so itemized deductions came to $24,984 instead of $35,717 and taxable income of $209,430 crossed into the 32% bracket. It also used 2025 bracket thresholds instead of 2026 ones, added the $1,750 refund, and skipped the 0.5% charitable floor. It omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488: the head's $3,744 is the same interest as the tax unit's $2,976 + $768, and the $750k acquisition-debt limit restricts the deduction to $2,808. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap and skipped the 0.5% charitable floor. It omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation and wrongly said itemized deductions are limited by the standard deduction. At 2026 brackets, $38,141 implies about $189,800 of taxable income, roughly $7,100 below the correct $196,918. It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"At 2026 brackets, $35,233 implies about $177,600 of taxable income, roughly $19,300 below the correct $196,918. Its reasoning invokes a retirement-savings credit that is unavailable at $232,635 of AGI, far above the Saver's Credit income limit. It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-sunset rules: a ~$5,300 personal exemption, $40,615 of itemized deductions, and pre-TCJA rates. The OBBBA made the TCJA brackets permanent with no personal exemption, and the correct 2026 rules give $39,858.42 on $196,918 of taxable income. It also computed NIIT at ~$70 on netted investment income instead of $163.40 on $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2026 brackets and charitable floor, but its taxable income of $210,286 is $13,368 too high. That matches leaving the ~$11,896 of Connecticut income tax out of SALT and including the $1,750 state refund. The excess pushed $8,511 into the 32% bracket, and it omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used $234,414 of AGI, which includes the $1,750 state refund, and itemized deductions of about $38,600 against the correct $35,717, landing near $195,800 of taxable income. It then computed NIIT at $70 on investment income netted for the farm rent loss, instead of 3.8% of $4,300 ($163.40)."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset and applied a restored personal exemption and pre-TCJA rates, but the OBBBA kept the 10/12/22/24/32% schedule permanent with no personal exemption. Its itemized deductions left out Connecticut income tax, leaving its $203,194 of taxable income about $6,300 above the correct $196,918. It omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, giving AGI of $226,985 instead of $232,635, and omitted the $163.40 NIIT on $4,300 of interest and rental income. Its $38,627.54 corresponds to about $191,800 of taxable income, roughly $5,100 below the correct $196,918."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It overstated itemized deductions at $40,221 against the correct $35,717. With mortgage interest at $2,808, its total implies about $37,400 of charitable gifts plus SALT, against the correct $32,909 ($19,001 after the 0.5% floor plus ~$13,908 SALT). It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It omitted the $163.40 NIIT on $4,300 of interest and rental income, even though MAGI exceeds the $200,000 threshold. Its taxable income of $196,143 is $775 below the correct $196,918: it added the $1,750 refund but overstated itemized deductions by more. Regular tax came to $39,672 instead of $39,858.42."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, giving AGI of $226,985 instead of $232,635, when only the $617 401(k) deferral reduces gross wages. Its taxable income of $190,999 is $5,920 short of the correct $196,918, which cost $1,420.68 of regular tax. It also omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 brackets, $39,250 corresponds to about $194,400 of taxable income, roughly $2,500 below the correct $196,918, so it overstated deductions. It also omitted the $163.40 NIIT on $4,300 of interest and rental income."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law: a $5,427 personal exemption, $2,966 of 2%-floor employee expenses, the full $3,744 of mortgage interest under the old $1M limit, and pre-TCJA brackets. The OBBBA made permanent the TCJA brackets, the $750k mortgage limit, and the repeal of personal exemptions and miscellaneous deductions. It also omitted NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $190,831 at restored pre-TCJA 10/15/25/28% brackets after a $5,437 personal exemption, but the OBBBA made the TCJA rate schedule permanent with no exemption. At the correct 2026 schedule, the $196,918 of taxable income yields $39,858.42, plus the $163.40 NIIT it omitted."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $20,164 of charitable gifts without the 2026 0.5%-of-AGI floor (the correct amount is $19,001). It overestimated Connecticut income tax, putting SALT at $15,371 instead of ~$13,908. It also computed NIIT on $1,853 after netting the farm rent loss, instead of on $4,300 ($163.40)."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025, adding $2,966 of 2%-floor miscellaneous deductions, deducting the full $3,744 of mortgage interest, and applying pre-TCJA brackets. The OBBBA made permanent the TCJA brackets, the $750k mortgage limit, and the elimination of miscellaneous deductions. Its $194,371 of taxable income yields about $39,250 at the actual 2026 schedule, so the pre-TCJA rates caused the overstatement; it also omitted NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of 2%-floor miscellaneous itemized deductions for unreimbursed employee business expenses, which the OBBBA permanently eliminated, and skipped the 2026 0.5%-of-AGI charitable floor. Those extra deductions cut regular tax to $39,214 instead of $39,858.42, while its $163 NIIT matched the $4,300 base."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate."
-us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 with no derivation. That ignores $196,918 of taxable income, which produces $39,858.42 of regular tax at 2026 single brackets plus $163.40 of NIIT, with no nonrefundable credit to offset any of it."
-us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $3,744 of mortgage interest without prorating for the $750,000 acquisition-debt limit on $1,000,000 of debt (only $2,808 is allowed). It added the $1,750 state refund to AGI and omitted the $163.40 NIIT on $4,300 of interest and rental income. These errors offset each other to land $20 high."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the $18,525 standard deduction instead of itemizing $35,717 ($19,001 of charitable gifts after the 0.5% floor, ~$13,908 of SALT, and $2,808 of mortgage interest), overstating taxable income by about $17,600. It also skipped the $617 401(k) exclusion, used 2025 bracket thresholds, and omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used married-filing-separately status for an unmarried single filer, added the full $20,164 of charitable gifts on top of the standard deduction instead of itemizing, and counted the $6,147 of tax-exempt pension as taxable. It also invented a $750 retirement income credit, and its reported $48,557 does not match its own $45,287 − $750 = $44,537."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached about $200,500 of taxable income but reported $30,238. The 2026 single brackets give about $40,700 on that amount. On the deduction side it also used the old $10,000 SALT cap instead of the full $13,908 allowed under the $40,400 OBBBA cap, took the full $3,744 of mortgage interest instead of $2,808 under the $750,000 debt limit, skipped the 0.5% charitable floor, and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI and skipped the $28 IRA deduction, which put taxable income at $198,819 instead of $196,918. It also figured NIIT as about $70 on $1,853 after netting the farm rent loss. The correct NIIT is 3.8% of the full $4,300 of interest and rental income, or $163.40."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It counted 100% of Social Security as income and took a $30,000 standard deduction instead of itemizing $35,717 (SALT $13,908, mortgage interest $2,808, charity $19,001). It then subtracted a made-up $11,297 of withholding, which never reduces tax before refundable credits. Its bracket tax of $29,119 on $210,670 is also far below the roughly $43,900 that the 2026 single schedule produces."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a $25,000 guess without doing any bracket calculation. AGI of $232,635 minus $35,717 of itemized deductions leaves $196,918 of taxable income, which is taxed at $39,858 under the 2026 single brackets, plus $163.40 of NIIT. The guess is about $15,000 too low."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket arithmetic came to $40,823, but it then reported $38,740 with no credit or adjustment to explain the cut. Its deductions capped SALT at the old $10,000 instead of the full $13,908 allowed under OBBBA's $40,400 cap. It also skipped the $750,000 mortgage-debt limit and the 0.5% charitable floor, and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the expired $10,000 limit instead of deducting the full ~$13,908 of CT income tax and property tax under OBBBA's $40,400 cap. It also skipped the 0.5%-of-AGI charitable floor and added the $1,750 refund to AGI, which left taxable income at $202,030 instead of $196,918. It also left out the $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the $2,012 property tax as SALT and left out the ~$11,896 of CT income tax. It also took the full $3,744 of mortgage interest instead of $2,808 under the $750k/$1M debt ratio and skipped the 0.5% charitable floor, which produced $208,465 of taxable income instead of $196,918. Its $40,650 is also below the 2026 bracket tax on its own taxable income (about $43,165), and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI ($234,385 instead of $232,635). That raised taxable income to $198,906 instead of $196,918 and added about $480 of regular tax. It also figured NIIT as $70 on $1,853 after netting the farm rent loss, when the correct NIIT is 3.8% of the $4,300 of interest and rental income, or $163.40."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-2018 rates (10/15/25/28/33%), a personal exemption, no SALT cap, and 2%-floor miscellaneous deductions. OBBBA made the TCJA brackets and the repeal of those items permanent. It also left CT income tax out of SALT, and its submitted $42,046 does not match its own final figure of $47,383."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $207,651 of taxable income but reported $24,500. The 2026 single brackets give about $42,900 on that amount. Its taxable income is too high because it left the ~$11,896 of CT income tax out of SALT, even though it is fully deductible under the $40,400 cap. It also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $235,031 includes the $1,750 state refund and does not take the $617 traditional 401(k) deferral out of wages; the correct AGI is $232,635. That pushed taxable income to $199,649 instead of $196,918. It also figured NIIT as $70 on $1,853 net instead of $163.40 on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer-sponsored insurance premiums from wages, but employment income is $164,980, reduced only by the 401(k) deferral. It also skipped the 0.5% charitable floor and understated SALT at $11,815. Its $34,322 is about $4,400 below the 2026 bracket tax on its own $192,198 of taxable income, and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-2018 brackets, a $5,063 personal exemption, and 2%-floor miscellaneous deductions. OBBBA made the TCJA rate schedule permanent and kept the personal exemption and miscellaneous itemized deductions repealed. It also subtracted the $7,429 of ESI premiums from wages and computed NIIT on $1,853 instead of $4,300."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $5,250 personal exemption with pre-TCJA brackets. OBBBA keeps the personal exemption at zero and the 10/12/22/24% schedule permanent. Its $42,059 of itemized deductions also exceeds the allowable $35,717, and it figured NIIT on $1,853 net instead of the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state refund to AGI ($234,414 instead of $232,635) and deducted the full $20,164 of charitable gifts without OBBBA's 0.5%-of-AGI floor, which allows $19,001. That gave $198,557 of taxable income instead of $196,918. It also left out the $163.40 net investment income tax on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, so it dropped the SALT cap, deducted $2,966 of 2%-floor miscellaneous employee expenses, took the full $3,744 of mortgage interest, and used pre-TCJA brackets. OBBBA made the TCJA brackets, the $750,000 mortgage limit, and the repeal of miscellaneous deductions permanent. The higher pre-TCJA rates produced $45,093, versus $39,858 of regular tax plus $163.40 of NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. The correct path is AGI of $232,635 minus $35,717 of itemized deductions, or $196,918 of taxable income, taxed at $39,858 under the 2026 single brackets, plus $163.40 of NIIT. Its $39,591 is $431 short, consistent with leaving out the NIIT and overstating deductions by about $1,100."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages and deducted 2%-floor miscellaneous employee expenses, which OBBBA permanently repealed. It then applied bracket rates heavier than the 2026 OBBBA schedule, producing $41,404 of regular tax on a taxable base below the correct $196,918. Its $87 of NIIT also falls short of 3.8% × $4,300 = $163.40."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a $5,000 personal exemption, $3,149 of 2%-floor miscellaneous deductions, uncapped mortgage interest, and pre-TCJA rate brackets. None of these apply under OBBBA's permanent extension. It also subtracted the $7,429 of ESI premiums from wages and figured NIIT as $70 on $1,853 instead of $163.40 on $4,300."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $33,814 is about $6,200 short, consistent with understating taxable income by about $25,000 at the 24% marginal rate and leaving out the NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages, giving AGI of $226,985 instead of $232,635. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under OBBBA's $40,400 cap and skipped the 0.5% charitable floor. It left out the $163.40 NIIT on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $37,824 is $2,198 short, consistent with taxable income about $8,500 too low and leaving out the NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $33,024 is about $7,000 short, consistent with understating taxable income by roughly $28,500 and leaving out the NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the $2,012 property tax as SALT and left out the ~$11,896 of CT income tax, which is fully deductible under OBBBA's $40,400 cap. It also skipped the 0.5% charitable floor, so itemized deductions came to $24,984 instead of $35,717 and taxable income to $209,430 instead of $196,918. It also added the $1,750 refund to AGI and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488 by adding the head's $3,744 to the tax unit's $2,976 + $768, and it ignored the $750,000 debt limit that cuts deductible interest to $2,808. It also capped SALT at the old $10,000 instead of ~$13,908 under the $40,400 cap and skipped the 0.5% charitable floor. It left out the $163.40 NIIT on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It did no calculation and treated deductions as limited by the standard deduction, even though itemized deductions of $35,717 are far above the $18,150 standard deduction. The correct $196,918 of taxable income gives $39,858 of regular tax plus $163.40 of NIIT. Its $38,141 is consistent with taxable income about $7,200 too low and leaving out the NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no actual calculation. The correct path is AGI of $232,635 minus $35,717 of itemized deductions, or $196,918 of taxable income, taxed at $39,858, plus $163.40 of NIIT. Its $35,233 is consistent with understating taxable income by about $19,000 and leaving out the NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: a $5,300 personal exemption, about $40,615 of itemized deductions including miscellaneous expenses, and pre-TCJA brackets. OBBBA made the TCJA schedule permanent, with no personal exemption and no miscellaneous itemized deductions. Its NIIT of about $70 also falls short of $163.40 on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $210,286 is $13,368 above the correct $196,918. That gap matches leaving the ~$11,896 of CT income tax out of the SALT deduction, which is fully deductible under OBBBA's $40,400 cap. Its $43,748 also leaves out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It figured NIIT as about $70 on investment income netted against the farm rent loss, instead of 3.8% of the full $4,300 of interest and rental income ($163.40). With AGI of $234,414, which includes the $1,750 refund, its regular tax of about $39,594 implies itemized deductions near $38,600. That is about $2,900 above the allowable $35,717 (SALT $13,908, mortgage interest $2,808, charity $19,001 after the 0.5% floor)."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, restored the personal exemption, and applied pre-TCJA rates. OBBBA made the 10/12/22/24% schedule and the zero personal exemption permanent. Its $48,134 is about $8,100 above the correct $39,858 of regular tax plus $163.40 of NIIT on $196,918 of taxable income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer-sponsored insurance premiums from wages, giving AGI of $226,985 instead of $232,635; employment income is $164,980, reduced only by the 401(k) deferral. That lowered taxable income by about $5,650. It also left out the $163.40 NIIT owed because MAGI is above $200,000."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It claimed $40,221 of itemized deductions, about $4,500 more than the allowable $35,717 (SALT $13,908, mortgage interest $2,808 after the debt limit, charity $19,001 after the 0.5% floor). It also added the $1,750 refund to AGI. It left out the $163.40 NIIT on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $234,414, which adds the $1,750 refund, and itemized deductions of about $38,271, roughly $2,550 above the allowable $35,717. That put taxable income at $196,143 instead of $196,918. Its bracket arithmetic is right for that figure, but it left out the $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages, putting AGI at $226,985 instead of $232,635 and taxable income at $190,999 instead of $196,918. It also left out the $163.40 NIIT on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $39,250 is $772 short, consistent with leaving out the NIIT and understating taxable income by about $2,500."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a $5,427 personal exemption, $2,966 of 2%-floor miscellaneous deductions, uncapped mortgage interest, and pre-TCJA brackets. OBBBA made the TCJA schedule, the $750,000 mortgage limit, and the repeal of the exemption and miscellaneous deductions permanent. It also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied restored 10/15/25/28% brackets and a $5,437 personal exemption on the assumption that the TCJA expired in 2026. OBBBA made the 10/12/22/24% schedule and the zero personal exemption permanent. It also left out the $163.40 NIIT on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $20,164 of charitable gifts without OBBBA's 0.5%-of-AGI floor, which allows $19,001, and overstated SALT at $15,371 instead of $13,908. Together these cut taxable income to $194,321 instead of $196,918. It also figured NIIT on $1,853 after netting the farm rent loss instead of on the $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-TCJA brackets, $2,966 of 2%-floor miscellaneous deductions, and uncapped mortgage interest. OBBBA made the TCJA rate schedule, the $750,000 mortgage limit, and the repeal of miscellaneous deductions permanent. It also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of unreimbursed employee business expenses as a 2%-floor miscellaneous itemized deduction, which OBBBA permanently eliminated, and it skipped the 0.5% charitable floor. Together these cut taxable income to $194,235 instead of $196,918. Its NIIT of $163 is correct, but its regular tax is about $640 too low."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $40,021.82 result."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $40,021.82 result."
+us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported zero federal tax even though taxable income is $196,918, which produces $39,858 of regular tax plus $163.40 of NIIT. No deduction or nonrefundable credit removes that liability."
+us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $3,744 of mortgage interest without applying the $750,000/$1,000,000 acquisition-debt ratio, which limits it to $2,808. It also added the $1,750 refund to AGI, leaving taxable income at $197,685 instead of $196,918. It left out the $163.40 NIIT owed on the $4,300 of interest and rental income because MAGI is above $200,000."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took an $18,525 standard deduction without comparing it to the $35,717 of itemized deductions (SALT $13,908, mortgage interest $2,808, charity $19,001). It also added the $1,750 refund, so taxable income came to $214,506 instead of $196,918. It used 2025 bracket thresholds, taxed the amount above $197,300 at 24% instead of 32%, and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It treated the head as married filing separately instead of single and added the $6,147 of tax-exempt pension to taxable pensions. It stacked charitable donations on top of the standard deduction instead of itemizing, and claimed a $750 retirement income credit that does not exist. Its submitted $48,557 also contradicts its own figure of $45,287 − $750 = $44,537."
us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -9542,93 +9776,95 @@ us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou
us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model returned 0 without applying Medicare's age-based eligibility rule. At age 76, the head exceeds the age-65 threshold, so the eligibility value is 1 regardless of the household's income, assets, or employer-sponsored coverage."
us,scenario_120,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right: $10,267.01 Social Security, $2,401.16 Medicare, $827.99 CT Paid Leave, total $13,496.16. It then submitted $13,736.63, a number its own work does not support and $240.47 above the total it had just computed."
-us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It got federal FICA right at $12,668.17 but left out Connecticut's mandatory 0.5% Paid Leave employee contribution on wages up to the Social Security wage base. That contribution is $827.98 and brings the total to $13,496.16."
-us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It used the old $168,600 wage base and got the Social Security ($10,267.14) and Medicare ($2,401.06) products slightly wrong. For the state piece it used a made-up −$48.79 'net from itemizations' adjustment. The actual CT Paid Leave contribution is 0.5% × $165,597 = $827.98, and adding it gives $13,496.16."
-us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said CT has no mandatory employee payroll tax and stopped at federal FICA of $12,668.17. It left out the CT Paid Leave employee contribution of 0.5% × $165,597 = $827.98."
-us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"It found all three pieces: $10,267 Social Security, $2,401 Medicare, and the 0.5% CT paid-leave tax of $828. It then threw away the correct sum of $13,496 and submitted $10,680, citing an 'employee payroll tax computation' adjustment that does not exist."
-us,scenario_120,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,It deliberately left out the CT paid-leave contribution on the false assumption that PolicyEngine does not model it. PolicyEngine includes ct_employee_paid_leave_contribution of $827.98 (0.5% of wages) in employee state payroll tax.
-us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It said Connecticut has no mandatory employee-side payroll tax. CT's Paid Leave program requires a 0.5% employee contribution on wages up to the Social Security wage base, which is $827.98 here, and the model left it out of the $12,668.17 FICA total."
-us,scenario_120,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare ($12,668.17) and explicitly added no state payroll tax. It missed Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98."
-us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It cut FICA wages to $157,551 by subtracting the $7,429 in employer insurance premiums and the $617 traditional 401(k) deferral. 401(k) elective deferrals are FICA wages, and PolicyEngine applies payroll tax to the full $165,597 of listed gross wages. It also said CT has no employee payroll tax and left out the $827.98 Paid Leave contribution."
-us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $7,429 in employer insurance premiums from wages and computed every piece, including the 0.5% CT Paid Leave contribution, on $158,168. PolicyEngine applies Social Security, Medicare, and the CT contribution to the full $165,597 of listed gross wages, which gives $13,496.16."
-us,scenario_120,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at federal FICA of $12,668.17 and did not add Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98."
-us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It brought in an Additional Medicare Tax adjustment that does not apply, because $165,597 of wages is under the $200,000 threshold. That pushed its answer to $14,197, well above the correct $10,267.01 Social Security + $2,401.16 Medicare + $827.98 CT Paid Leave = $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It used $158,168 of FICA wages by subtracting the $7,429 in employer insurance premiums. PolicyEngine applies the 6.2%, 1.45%, and 0.5% CT Paid Leave rates to the full $165,597 of listed gross wages."
-us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced FICA-taxable wages to $158,168 by subtracting the $7,429 in employer insurance premiums, which pulled every component down. The correct base for Social Security, Medicare, and the CT PFML contribution is the full $165,597 of listed gross wages."
-us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"It said it used only Social Security and Medicare on wages, but $14,352.19 is well above what those rates give on $165,597 ($12,668.17). It therefore applied them to an inflated base or wrong rates, and it also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It computed Social Security, Medicare, and the 0.5% CT Paid Leave contribution on $158,168, having subtracted the $7,429 in employer insurance premiums. PolicyEngine applies all three to the full $165,597 of gross wages."
-us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 is the three-part total on $158,168: gross wages minus the $7,429 in employer insurance premiums. PolicyEngine does not reduce the $165,597 wage input by those premiums for payroll tax."
-us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It applied the right rates, but its $12,891.10 matches computing them on $158,168, which is gross wages minus the $7,429 in employer insurance premiums. The correct base is the full $165,597, which gives $13,496.16."
-us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the traditional 401(k) deferral and the employer insurance premiums, then also subtracted the $109 Roth 401(k) contribution, which is after-tax, ending at $157,442. FICA applies to the full $165,597 of wages. It also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It said CT has no employee payroll tax and reported only federal FICA of $12,668. It missed the mandatory 0.5% CT Paid Leave employee contribution of $827.98."
-us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It said CT has no mandatory employee payroll tax, which drops the $827.98 Paid Leave contribution. Its $10,087 is also below the Social Security component alone (6.2% × $165,597 = $10,267.01), so it dropped the $2,401.16 of Medicare tax and misapplied the Social Security rate."
-us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It stated the right rates, 6.2% plus 1.45% on wages under the base, but submitted $6,180, less than half of 7.65% × $165,597 = $12,668.17. It also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It computed the CT paid-leave contribution as 0.5% of the $184,500 Social Security wage cap ($922.50) rather than 0.5% of actual wages. The cap only limits covered earnings, so the contribution is 0.5% × $165,597 = $827.98."
-us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $617 traditional 401(k) deferral to get $164,980 of FICA wages. Elective deferrals stay in Social Security and Medicare wages under IRC §3121(v)(1)(A). It also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used $158,168 of payroll-taxable wages by subtracting the $7,429 in employer insurance premiums. PolicyEngine applies Social Security, Medicare, and the 0.5% CT paid-leave contribution to the full $165,597 of gross wages."
-us,scenario_120,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $7,429 in employer insurance premiums as pretax and cut the payroll-tax base to $158,168. PolicyEngine applies the 6.2%, 1.45%, and 0.5% CT paid-leave rates to the full $165,597 of listed gross wages, which gives $13,496.16."
-us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only the standard Social Security and Medicare rates to gross wages ($12,668). It left out Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98."
-us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It said CT has no employee state payroll tax and stopped at 7.65% × $165,597 = $12,668. It missed the CT Paid Leave employee contribution of 0.5% of wages, $827.98."
-us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It said there are 'no other employee payroll taxes' beyond federal FICA. Connecticut requires a 0.5% Paid Leave employee contribution, $827.98 here, which the model left out."
-us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"It said CT has no mandatory employee state payroll tax. The CT Paid Leave program requires a 0.5% employee contribution on wages up to the Social Security base, $827.98, which is missing from its $12,668."
-us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"It submitted $0 despite $165,597 of wages, applying no payroll tax at all. The correct total is $10,267.01 Social Security + $2,401.16 Medicare + $827.98 CT Paid Leave = $13,496.16."
-us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It said CT has no mandatory employee payroll tax and reported only federal FICA of $12,668. It missed the 0.5% CT Paid Leave employee contribution of $827.98."
-us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It got Social Security and Medicare right ($12,668.17) but said CT has no mandatory employee state payroll tax. That left out the $827.98 Paid Leave contribution, which is 0.5% of covered wages."
-us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It treated employee Social Security tax as capped at $9,655. The 2026 wage base of $184,500 is above the $165,597 of wages, so the tax is the full 6.2% × $165,597 = $10,267.01. It also dropped the $2,401.16 of Medicare tax and the $827.98 CT Paid Leave contribution."
+us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model's own reasoning had the correct parts: SS $10,267.01, Medicare $2,401.16, CT PFML 0.5% ≈ $828, total $13,496.16. It then submitted $13,736.63, which doesn't follow from that arithmetic. The extra $240.47 has no basis in any rule."
+us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model got SS ($10,267.01) and Medicare ($2,401.16) right. It left out Connecticut's mandatory employee Paid Leave contribution, 0.5% × $165,597 = $827.98, which counts as an employee state payroll tax."
+us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used the 2024 wage base ($168,600) and slightly miscalculated SS and Medicare. It made up a 0.64% Connecticut payroll rate and then swapped it for a meaningless '-$48.79 net from itemizations'. Connecticut's employee tax is the 0.5% Paid Leave contribution on $165,597, or $827.98, and income-tax itemizations have no effect on payroll tax."
+us,scenario_120,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"The model rounded FICA down to about $12,000 when 7.65% × $165,597 is exactly $12,668.17. It also left out Connecticut's 0.5% employee Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model said Connecticut has no mandatory employee payroll tax and stopped at FICA ($12,668.17). Connecticut charges employees a 0.5% Paid Leave contribution, here $827.98, which brings the total to $13,496.16."
+us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model listed all three correct parts: SS $10,267, Medicare $2,401, CT paid leave $828, which add to $13,496. It then submitted $10,680, a number that matches no combination of them, citing an unexplained 'PolicyEngine computation' adjustment."
+us,scenario_120,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"The model knowingly left out the Connecticut paid-leave contribution on the assumption that PolicyEngine doesn't model it. The engine includes the 0.5% employee contribution ($827.98) as a state payroll tax, so the total is $13,496.16, not $12,668."
+us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model said Connecticut has no employee-side payroll tax because it lacks SDI/UI withholding. That overlooks the CT Paid Leave contribution of 0.5% of wages ($827.98).
+us,scenario_120,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model counted only federal FICA ($12,668.17) and explicitly added no state payroll tax. It missed Connecticut's mandatory 0.5% employee Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model shrank FICA wages to $157,551 by subtracting the $7,429 ESI premium and the $617 traditional 401(k) deferral. 401(k) deferrals are FICA wages, and the facts give gross wages with no stated salary-reduction premium, so the base is $165,597. It also said Connecticut has no employee payroll tax and left out the $827.98 Paid Leave contribution."
+us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model included the CT 0.5% paid leave contribution but applied all three rates to $158,168, after subtracting the $7,429 ESI premium. The base is the full $165,597 gross wage, which gives $10,267.01 + $2,401.16 + $827.98 = $13,496.16."
+us,scenario_120,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model stopped at SS plus Medicare ($12,668.17) and left out Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model brought in Additional Medicare Tax 'adjusted for this individual' even though wages of $165,597 are below the $200,000 threshold. Its $14,197 is roughly 8.55% of wages, consistent with adding 0.9% to all wages. It also never included the CT 0.5% Paid Leave contribution."
+us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model applied 6.2% + 1.45% + 0.5% to $158,168, after subtracting the $7,429 ESI premium. The facts give gross wages of $165,597 and don't describe the premium as a pre-tax salary reduction, so the base is $165,597 and the total is $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 ESI premium to get a $158,168 FICA/PFML base. Applying the same rates to the full $165,597 gives SS $10,267.01, Medicare $2,401.16 and CT PFML $827.98, for $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model said it counted only SS and Medicare on wages, but 7.65% × $165,597 is $12,668.17. Its $14,352.19 implies a base of about $187,600, more than the wages. It also left out Connecticut's $827.98 Paid Leave contribution."
+us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model computed SS, Medicare and CT Paid Leave on $158,168, after subtracting the ESI premium, instead of the stated $165,597 gross wages. That understated every component."
+us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The $12,891 answer equals 8.15% of $158,168, so the model took the $7,429 ESI premium out of the payroll base. The base is the full $165,597 gross wage, which gives $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model had the right three taxes, but its $12,891.10 matches a base of about $158,168, i.e. wages less the ESI premium. The base is the full $165,597, which gives $13,496.16."
+us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model subtracted the traditional 401(k) deferral, the ESI premium and even the $109 Roth 401(k) contribution from FICA wages. Both 401(k) types are FICA wages, and the premium isn't stated as a salary reduction. It also left out Connecticut's 0.5% Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model said Connecticut has no employee payroll tax and stopped at FICA ($12,668). It missed the mandatory 0.5% CT Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The $10,087 answer is below even the Social Security tax alone ($10,267.01). The model effectively dropped the $2,401.16 Medicare tax and also wrongly denied the CT employee Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly named the 6.2% and 1.45% rates on wages below the cap, but its sum of $6,180 is an arithmetic error. 7.65% × $165,597 is $12,668.17. It also left out the CT 0.5% Paid Leave contribution."
+us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model computed the CT paid-leave contribution as $922.50, which is 0.5% of the $184,500 Social Security wage cap rather than of actual wages. The cap only limits the contribution when wages exceed it, so the correct figure is 0.5% × $165,597 = $827.98."
+us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model subtracted the $617 traditional 401(k) deferral from FICA wages, giving a $164,980 base. Elective 401(k) deferrals are still Social Security and Medicare wages. It also left out Connecticut's 0.5% Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model applied SS, Medicare and CT paid-leave rates to $158,168, after subtracting the ESI premium. The facts give $165,597 gross wages with no pre-tax salary reduction, so the total is $13,496.16."
+us,scenario_120,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model treated the $7,429 employer health premium as a pre-tax exclusion and taxed $158,168. All three payroll taxes apply to the full $165,597 gross wage."
+us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model counted only federal FICA ($12,668). It left out Connecticut's mandatory employee Paid Leave contribution, 0.5% of wages or $827.98."
+us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model said Connecticut has no employee state payroll tax. CT charges a 0.5% employee Paid Leave contribution, here $827.98, which brings the total to $13,496.16."
+us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model concluded there were 'no other employee payroll taxes' after FICA. It missed Connecticut's mandatory 0.5% Paid Leave contribution of $827.98.
+us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"The model said Connecticut has no mandatory employee state payroll tax. It overlooked the CT Paid Leave contribution, 0.5% × $165,597 = $827.98."
+us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value or explanation, so there was no answer to score."
+us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value or explanation, so there was no answer to score."
+us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"The model reported $0 payroll tax for someone with $165,597 in wages. Those wages are subject to 6.2% Social Security, 1.45% Medicare and Connecticut's 0.5% Paid Leave contribution, which total $13,496.16."
+us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model said Connecticut has no mandatory employee payroll tax and stopped at $12,668. It missed the 0.5% CT Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model computed SS and Medicare correctly but said Connecticut has no mandatory employee payroll tax. It left out the $827.98 CT Paid Leave contribution.
+us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model treated Social Security tax as 'capped at $9,655'. Wages of $165,597 are below the 2026 wage base, so the full 6.2% applies ($10,267.01). It also dropped the $2,401.16 Medicare tax and the $827.98 CT Paid Leave contribution."
us,scenario_120,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It never applied Connecticut's high-income Social Security rule (subtract taxable benefits minus 25% of total benefits = $25,140) and guessed a 'limited' subtraction instead. It also added back $2,852 of tax-exempt interest that the computation does not add. Its $12,130 is also more than the 2026 schedule plus $680 of add-back and recapture produce on its own ~$205,000 base (about $11,755), so its bracket arithmetic is wrong too."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied the correct $25,140 Social Security subtraction. However, it started from a federal AGI of $234,414 that includes the $1,750 state refund and skips the $29 IRA deduction, overstating CT AGI by $1,779 (+$116 of tax). It added the $250 2% phase-out add-back and the $180 middle recapture but left out the separate $250 low-rate benefit recapture, which leaves it $134 short net."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It invented Connecticut deductions that do not exist: federal income tax paid, federal itemized deductions and an over-65 deduction. Connecticut taxes CT AGI of $207,495 (federal AGI less the $25,140 Social Security subtraction) with the personal exemption fully phased out, which gives $11,917, not $0."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It said no Social Security subtraction applies above $75,000, missing that Connecticut still subtracts taxable benefits above 25% of total benefits ($25,140). It also added back $2,852 of tax-exempt interest, which inflated CT AGI to $236,883 instead of $207,495. It then used a 3% first-bracket rate instead of 2% and arbitrarily cut its own $13,247 to $11,890 rather than adding the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It never computed the $25,140 Social Security subtraction and called it only 'partial'. It used the pre-2024 3%/5% bottom rates and left out the 6.5% bracket above $200,000. Its $9,970 is below even the graduated tax on its own ~$200,000 base ($11,050 at those rates), and it included neither the $250 phase-out add-back nor the $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the Social Security subtraction phases out to zero but then used a ~$198,800 base. It described the schedule as 2%–5.5%, dropping the 6% ($100k–$200k) and 6.5% brackets. The correct computation taxes $207,495 at 2/4.5/5.5/6/6.5% ($11,237.18) and adds $680 of phase-out add-back and recapture, not the $9,450 it reported."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It got CT AGI ($207,495) and the graduated tax ($11,237) right and added the $250 2% phase-out add-back and $180 middle recapture. It left out the separate $250 low-rate benefit recapture for single filers above $200,000, so it is exactly $250 short."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It concluded there is no Social Security subtraction above $100,000. That misses Connecticut's rule that high-income filers still subtract taxable benefits above 25% of total benefits ($35,615 − $10,475 = $25,140). It also kept the $1,750 refund in AGI, so it taxed $234,414 instead of $207,495. It treated the graduated tax as final and left out the $250 2% phase-out add-back and the $430 benefit recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted non-deductible Roth contributions and never quantified the $25,140 Social Security subtraction. Its $7,200 is far below what its own 3%–6.99% schedule gives on its ~$198,000 base (about $10,900). The correct result taxes $207,495 at the 2026 rates ($11,237.18) plus $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It used a CT AGI of $209,891, about $2,400 above the correct $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction). It added only a $180 recapture and left out both the $250 2% phase-out add-back and the $250 low-rate benefit recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security. Above the single threshold, Connecticut subtracts only the $25,140 above 25% of benefits. It also added back $2,852 of tax-exempt interest and applied the $15,000 personal exemption, which is fully phased out at this income. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It exempted Social Security in full instead of limiting the subtraction to taxable benefits above 25% of total benefits ($25,140), and it used the pre-2024 3%/5% bottom rates. It subtracted a $1,549 personal tax credit that is fully phased out for a single filer at CT AGI of $207,495. It also left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It removed the refund ($232,664) but made no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. It used the pre-2024 3%/5% bottom rates ($11,050 through $200,000 instead of $10,750) and left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of only the $25,140 above 25% of benefits, which put CT AGI at $197,049 instead of $207,495. It then used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It inverted Connecticut's Social Security rule, subtracting only 25% of taxable benefits ($8,904). The rule subtracts taxable benefits in excess of 25% of total benefits ($35,615 − $10,475 = $25,140), so its CT AGI of $223,731 is too high. It also started the recapture at a nonexistent $181,700 threshold and added about $1,020, instead of the $250 add-back plus $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. The correct path taxes CT AGI of $207,495 (federal AGI less the $25,140 Social Security subtraction) at 2/4.5/5.5/6/6.5% ($11,237.18) plus $680 of add-back and recapture. Its $7,800 matches a ~$152,000 base on the 2026 schedule, consistent with exempting all Social Security and pension income (which Connecticut taxes at this AGI) and skipping the add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It said high income eliminates the Social Security subtraction and taxed roughly the full federal AGI. Connecticut still subtracts taxable benefits above 25% of total benefits ($25,140) for filers over the threshold, which makes CT AGI $207,495, not about $232,000–$234,000."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It understated federal AGI at $225,235 (should be $232,635) and used a $26,711 Social Security subtraction instead of $25,140, which put CT AGI at $198,524 instead of $207,495. It then used the pre-2024 3%/5% bottom rates ($10,961 base) and added only $150 of recapture instead of the $250 phase-out add-back plus $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. The correct path taxes CT AGI of $207,495 at the 2026 schedule ($11,237.18) and adds the $250 2% phase-out add-back plus $430 benefit recapture ($250 low + $180 middle). Its $11,770 is $147 short, consistent with a roughly correct base but an under-count of the $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CT taxable income of $212,320, about $4,825 above the correct $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction). That extra income taxed at 6.5% (+$314) accounts for its $294 overstatement."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $13,125 matches the 2026 graduated tax on roughly the full $232,635 federal AGI, consistent with skipping Connecticut's $25,140 Social Security subtraction (taxable benefits above 25% of total benefits). The correct base is $207,495, taxed at $11,237.18 plus $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $9,706.75 matches a ~$182,600 base on the 2026 schedule, consistent with exempting all Social Security and also taking a $15,000 exemption that is fully phased out at this income. Connecticut subtracts only the $25,140 of taxable benefits above 25% of total benefits, so the base is $207,495 and the tax is $11,237.18 plus $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It fully exempted Social Security. Above the $75,000 single threshold, Connecticut subtracts only taxable benefits above 25% of total benefits ($25,140), leaving $10,475 taxable. It also took a $15,000 'standard deduction' that Connecticut does not have and a $200 property tax credit that is fully phased out at this income, and it left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 allowed above the income threshold. It then applied a flat 4.5% rate to $197,049, ignoring Connecticut's graduated 2/4.5/5.5/6/6.5% schedule and the $680 of phase-out add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $4,414 is less than the graduated tax on the first $100,000 alone ($4,750), consistent with taxing under half of the correct $207,495 CT AGI. Connecticut subtracts only the $25,140 of Social Security above 25% of benefits and allows no exemption or credit at this income, which gives $11,237.18 plus $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It relied on itemized deductions, which Connecticut does not allow. CT AGI is federal AGI ($232,635) less the $25,140 Social Security subtraction ($207,495), with the personal exemption fully phased out. Its $7,929 matches a ~$153,000 base on the 2026 schedule and leaves out the $680 of phase-out add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed the Social Security subtraction as 75% of taxable benefits ($26,711) instead of taxable benefits minus 25% of total benefits ($25,140). It also added back $2,852 of tax-exempt interest and skipped the $29 IRA deduction, putting CT AGI at $208,805 instead of $207,495. Its $680 of phase-out add-back and recapture is correct, so the $1,310 base overstatement taxed at 6.5% accounts for the whole $85 overshoot."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $235,031 with no Social Security subtraction, missing that Connecticut still removes taxable benefits above 25% of total benefits ($25,140) for filers over the threshold. It also applied an oversized high-income recapture instead of the $250 phase-out add-back plus $430 recapture that apply at CT AGI of $207,495."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $234,000 of CT AGI with no Social Security subtraction. Connecticut subtracts the $25,140 of taxable benefits above 25% of total benefits, which puts CT AGI at $207,495, and at that level the add-back plus recapture total $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It said Social Security relief is unavailable and taxed $234,414, missing the $25,140 subtraction of taxable benefits above 25% of total benefits. Its $10,895 is also below the graduated tax on its own base ($12,987), so its bracket arithmetic is wrong too."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It over-subtracted Social Security, reaching CT taxable income of $200,095 instead of $207,495. The subtraction is federally taxable benefits ($35,615) minus 25% of total benefits ($10,475), which is $25,140. The understated base cost $481 of tax at 6.5% and cut the middle recapture to $90 instead of $180."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $234,414 with no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. It then added about $2,250 of recapture on top of the $12,987 graduated tax, far more than the $250 phase-out add-back plus $430 recapture that apply at CT AGI of $207,495."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It removed only the state refund and made no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. Its $13,286.91 equals the pre-2024 3%/5% schedule applied to $234,414, not even to its stated $232,664 base."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It over-subtracted Social Security, reaching CT taxable income of $200,095 instead of $207,495. The subtraction is taxable benefits ($35,615) minus 25% of total benefits ($10,475), which is $25,140. Its $500 add-back matched the combined $250 phase-out add-back and $250 low recapture, but the understated base lost $481 at 6.5% and cut the middle recapture from $180 to $90."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $7,850 matches a ~$152,000 base on the 2026 schedule, consistent with exempting all Social Security and pension income. At this income Connecticut exempts only the $25,140 of taxable Social Security above 25% of benefits and no pension income, so CT AGI is $207,495 and the tax is $11,237.18 plus $680 of add-back and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 allowed above the threshold (taxable benefits minus 25% of total benefits), understating CT AGI by about $10,450 at $197,049. It also left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all taxable Social Security plus a 100% pension exemption. The pension/annuity subtraction is fully phased out for a single filer at this income, and the Social Security subtraction is limited to $25,140, so CT AGI is $207,495, not $166,833. It also used the pre-2024 3%/5% bottom rates and left out the $430 benefit recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It taxed federal AGI plus $2,852 of tax-exempt interest ($235,516) with no Social Security subtraction, missing the $25,140 subtraction. Its $11,050 base at $200,000 reflects the pre-2024 3%/5% bottom rates instead of $10,750, and it left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140 and added back $2,852 of tax-exempt interest. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It made no Social Security subtraction and taxed $234,385 instead of $207,495. It subtracted a $300 property tax credit that is fully phased out for a single filer at this income, and it left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so this output is missing. The correct derivation taxes CT AGI of $207,495 for $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so this output is missing. The correct derivation taxes CT AGI of $207,495 for $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned $0 with no reasoning. Connecticut taxes CT AGI of $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction) with no exemption or credit available at this income. A $0 answer is consistent with treating an elderly, disabled filer as exempt from Connecticut income tax, and no Connecticut rule provides that."
-us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It denied the Social Security subtraction; Connecticut still subtracts the $25,140 of taxable benefits above 25% of total benefits. It also invented a $15,476 charitable itemized deduction, which Connecticut does not allow, and it left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It left Social Security out of federal AGI and then subtracted the full $41,900 again. It also subtracted $6,147 of tax-exempt pension that was never in AGI and applied a $15,000 personal exemption that is fully phased out, cutting taxable income to $136,825 instead of $207,495. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its pre-credit tax of $11,943 is within $26 of the correct $11,917. It then subtracted a $500 property tax credit, but that credit is capped at $300 and fully phased out for a single filer at CT AGI of $207,495, so no nonrefundable credit applies."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It never quantified the Social Security modification (taxable SS minus 25% of benefits = $25,140 subtraction) and added back $2,852 of tax-exempt interest, which is not a CT addition here. Its asserted ~$205,000 base does not reproduce $12,130 under the schedule; the correct $207,524 CT AGI gives $11,239 plus $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied the $25,140 Social Security subtraction correctly but did not subtract the $1,750 state tax refund, which overstated CT AGI at $209,274 instead of $207,524. It also included only the $250 phase-out add-back and $180 middle-tax recapture, leaving out the $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented a deduction for federal income tax paid, federal itemized deductions and an over-65 deduction to bring CT tax to zero. Connecticut allows none of these: its tax is computed on CT AGI of $207,524 with no itemized or age deduction, which yields $11,919."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a bare $7,000 guess with no computation. The correct path is federal AGI $234,414 minus the $25,140 Social Security subtraction and the $1,750 refund, giving $207,524. Graduated tax on that is $11,239, plus $680 of add-back and recapture. The $7,000 figure is consistent with treating most income as exempt and leaving out the upper brackets."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It wrongly asserted that no Social Security subtraction applies above $75,000. In fact Connecticut caps taxable benefits at 25% for such filers, which is a $25,140 subtraction. It also skipped the refund subtraction and added tax-exempt interest, reaching $236,883. It then cut its $13,247 figure to $11,890 with no identified credit."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the pre-2024 3%/5% bottom brackets and capped the schedule at 6% even though income exceeds $200,000, where the 6.5% bracket applies. Its $9,970 is also below what its own brackets give on $200,000 ($11,050), and it never added the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It cut AGI by roughly the full $35,600 of taxable Social Security instead of the $25,140 excess over 25% of benefits, reaching $198,800. It then applied a 2%–5.5% schedule without the 6% and 6.5% brackets, which understated tax well below the $11,239 base plus $680 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its CT AGI (~$207,495) and graduated tax (~$11,237) were essentially correct. However, it added only $250 + $180 of recapture and omitted the separate $250 low-tax recapture, so the total add-back is $680, not $430."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded that Connecticut gives no Social Security relief above $100,000 AGI, so it left out the $25,140 subtraction (taxable SS above 25% of benefits). It also skipped the $1,750 refund subtraction and taxed $234,414. It then omitted the $250 phase-out add-back and $430 of benefit recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It treated Social Security as largely exempt and subtracted Roth contributions. It also applied a personal exemption and credit that are fully phased out at this income. The result is $7,200, far below the graduated tax of $11,239 on $207,524 CT AGI, plus $680 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Its CT AGI of $209,891 is $2,367 too high because it did not subtract the $1,750 state tax refund. It also added only $180 of recapture, leaving out the $250 phase-out add-back and the $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 excess over 25% of benefits. It also took a $15,000 personal exemption that is fully phased out for single filers at this AGI. Finally, it left out the $680 of phase-out add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It fully exempted Social Security instead of taxing 25% of benefits. It also applied a $1,549 personal tax credit, which is zero for a single filer at $207,524 CT AGI. The $680 of add-back and recapture was left out as well."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted only the state refund and left out the $25,140 Social Security subtraction. Connecticut caps taxable benefits at 25% for filers above $75,000 AGI, so CT AGI was overstated at $232,664 instead of $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security rather than $25,140, giving $197,049 instead of $207,524. It also used the old 3%/5% bottom brackets in place of 2%/4.5%. The $680 of phase-out add-back and benefit recapture was omitted."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It inverted the Social Security modification by subtracting only 25% of taxable SS ($8,904). The rule actually subtracts everything above 25% of benefits, which is $25,140. This overstated CT AGI at $223,731."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave a $7,800 figure with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. A result this low is consistent with exempting all Social Security and pension income and leaving out the upper brackets and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It explicitly denied any Social Security exemption at high income. Connecticut in fact subtracts taxable benefits above 25% of benefits ($25,140) at every income level. Taxing near-full federal AGI overstated the result."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It understated federal AGI at $225,235 instead of $234,414, over-subtracted Social Security ($26,711 instead of $25,140), and skipped the refund subtraction, ending at $198,524 CT AGI. It also added only $150 of the $680 add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no computation, and its $11,770 is $149 short of the correct $11,239 base tax plus $680 of add-back and recapture on $207,524 CT AGI. That gap is consistent with leaving out part of the benefit recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It subtracted only $22,094 from federal AGI instead of the $25,140 Social Security subtraction plus the $1,750 refund ($26,890 total). This overstated CT taxable income at $212,320. It also added $660 of recapture instead of $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation, and its $13,125 overshoots by $1,206. That is consistent with taxing CT AGI well above $207,524 by skipping most of the $25,140 Social Security subtraction (taxable SS above 25% of benefits)."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It gave no computation, and its $9,706.75 falls $2,212 short of the correct $11,919. That is consistent with exempting all taxable Social Security instead of only the amount above 25% of benefits, and leaving out the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It removed Social Security entirely and took a $15,000 'standard deduction' that Connecticut does not have. It also claimed a $200 property tax credit, which is fully phased out at this AGI, and left out the 6.5% bracket and $680 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140. It then applied a flat 4.5% rate to the whole $197,049 rather than the graduated 2%–6.5% schedule, and left out the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave $4,414 with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. A result this low is consistent with treating most income as exempt or deductible."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced income by itemized deductions, which Connecticut does not allow because it taxes CT AGI directly. This understated tax well below the $11,239 graduated amount on $207,524 plus $680 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It subtracted 75% of federally taxable Social Security ($26,711) rather than taxable SS minus 25% of benefits ($25,140). It also added back $2,852 of tax-exempt interest that is not a CT addition here, which overstated taxable income at $208,805 instead of $207,524. Its $680 of add-back and recapture was correct."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed about $235,031 with no Social Security subtraction and no refund subtraction. It also applied about $1,240 of recapture instead of $680. The correct CT AGI after the $25,140 SS and $1,750 refund subtractions is $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied the schedule to about $234,000 and left out the $25,140 Social Security subtraction (taxable SS above 25% of benefits) and the $1,750 refund subtraction. That put CT AGI about $26,500 too high."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It denied Social Security relief and taxed the full $234,414, ignoring the $25,140 SS subtraction and the refund subtraction. Its $10,895 is also inconsistent with its own base, since the graduated schedule on $234,414 gives about $12,987 before recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It cut CT taxable income to $200,095, about $7,429 below the correct $207,524, by over-subtracting beyond the $25,140 Social Security and $1,750 refund modifications. It also applied $590 of add-back and recapture instead of $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed the full $234,414 federal AGI with no Social Security or refund subtraction. It also inflated recapture far beyond $680. The correct CT AGI after the $26,890 of subtractions is $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted only the state refund, reaching $232,664, and left out the $25,140 Social Security subtraction that limits taxable benefits to 25% for filers above $75,000 AGI."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It reduced CT taxable income to $200,095, about $7,429 below the correct $207,524, by subtracting more than the $25,140 Social Security and $1,750 refund modifications. It also computed only $90 of recapture where the correct middle-tax recapture is $180 and the low-tax recapture is $250, a total of $590 against the correct $680."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave $7,850 with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. The low figure is consistent with exempting all Social Security and pension income and leaving out the top brackets and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140, the amount above 25% of benefits. That put CT AGI at $197,049 instead of $207,524. It also left out the $250 phase-out add-back and $430 of benefit recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It subtracted all taxable Social Security and applied a 100% pension exemption, which Connecticut phases out entirely for single filers at this AGI. It also used the old 3%/5% brackets and added only $200 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added tax-exempt interest and left out both the $25,140 Social Security subtraction and the $1,750 refund subtraction. It then used the old 3%/5% bracket base of $11,050 instead of $10,750 and omitted the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security and added tax-exempt interest, skipping the refund subtraction. It then applied the outdated 3%/5% bottom brackets and left out the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It taxed $234,385 with no Social Security or refund subtraction and left out the $680 of add-back and recapture. It also subtracted a $300 property tax credit that is fully phased out at this AGI."
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 with no reasoning, which implies no Connecticut liability at all. The correct path is CT AGI of $207,524 after the $25,140 Social Security and $1,750 refund subtractions, taxed at $11,239 plus $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It claimed the Social Security exclusion phases out entirely and invented a Connecticut charitable itemized deduction, which Connecticut does not have. It taxed $218,938 instead of $207,524 and left out the $680 of add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded all Social Security, subtracted tax-exempt pension income that was never in AGI, and took a $15,000 personal exemption that is phased out at this income. It also used old 3%/5% brackets. Together these understated CT taxable income at about $136,825 instead of $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It applied the married-filing-separately schedule instead of the single schedule. It also subtracted a $500 property tax credit, which exceeds the $300 maximum and is fully phased out at this AGI."
us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -9714,97 +9950,99 @@ us,scenario_121,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambigu
us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model taxed $38,709 of gross income less a $15,100 standard deduction, never applying the $32,600 medical expense itemized deduction that after the 7.5%-of-AGI floor exceeds that standard deduction by roughly $14,000 and zeroes the SC base. It compounded this with a fabricated bracket schedule containing 6.4% and 6.5% tiers (SC has only a zero-rate bracket, a 3% bracket, and one top rate) and then cut its own $1,395 result to $574 with an unquantified ""exemptions/credits"" adjustment it never identified."
us,scenario_121,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model used the correct 2026 federal standard deduction of $16,100 and correctly kept disability benefits out of the base, but stated that ""no SC subtractions"" apply and never itemized: the $32,600 of medical expenses less the 7.5% floor exceeds its own $25,665 AGI, so federal taxable income is $0, not $9,565. With a zero starting point, SC's 0%/3% schedule produces $0 rather than $180."
us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model reduced wages by a $14,600 standard deduction while ignoring the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor eliminates taxable income entirely. It then applied an invented four-tier SC schedule (0%/3%/4%/5.5%) that does not exist — SC has a zero-rate bracket, a 3% bracket, and a single top rate — and its $94.26 does not even follow from that schedule, which would yield about $314 on its own $11,065 base."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning stacked the $20,200 standard deduction and the $6,000 senior deduction but left out the 2026 $1,000 charitable deduction for non-itemizers. That gave $44,446 of taxable income and $4,867 of tax, $120 above the correct figure. It then submitted $6,552, a number its own computation does not support."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It taxed only 50% of Social Security ($16,000) when provisional income of $61,459 makes the full 85% ($27,200) taxable. It also invented a $21,700 base and a 'disabled' additional standard deduction, and it omitted the $6,000 senior and $1,000 non-itemizer charitable deductions. Its final $1,065 does not even follow from its own $3,668 tentative tax."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction (basic plus age and blind amounts). It omitted the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income came out at $50,446 instead of $43,446.18. That inflated ordinary-rate tax and pushed part of the qualified dividends into the 15% band."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used 2025-level standard deduction amounts ($15,000 base plus $2,000 each for age and blindness) and a rounded 'about $25,000' total, and it omitted the $1,000 non-itemizer charitable deduction. It then cut taxable income to about $40,000 with no basis and reported $1,971, which is far below the 10%/12% bracket tax on any taxable income in that range (about $4,500)."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of $43,546 was close to the correct figure despite an overstated $17,000 base standard deduction and the missing $1,000 charitable deduction. It then misapplied the rate schedule: $2,336 of tax on $41,722 of ordinary income is impossible under 2026 brackets (10% to $12,400, 12% to $50,400), which give about $4,759. It also charged $273 on qualified dividends that sit entirely in the 0% band."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, which OBBBA prevented by making the TCJA structure permanent. Under that wrong assumption it used an $8,300 pre-TCJA standard deduction, itemized with a 2%-floor miscellaneous deduction for employee business expenses, subtracted a $5,300 personal exemption, and applied 15%/25% brackets. The correct law gives $27,200 of standard, senior and non-itemizer charitable deductions and 10%/12% brackets."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of gross income, putting AGI at $68,822 instead of $70,646. It then deducted only the $20,200 standard deduction, omitting the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, which overstated taxable income at $48,622."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $20,200 standard deduction and the $6,000 senior deduction right. It missed the $1,000 charitable deduction that OBBBA gives single non-itemizers starting in 2026, so its taxable income was $44,446 instead of $43,446.18 and its tax was $120 (12% of $1,000) too high."
-us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used an understated $19,200 standard deduction and omitted both the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That produced $51,446 of taxable income instead of $43,446.18 and pushed qualified dividends into the 15% band."
-us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules that OBBBA prevented: it itemized with a 2%-floor miscellaneous deduction and state income tax, took a $4,840 personal exemption, and taxed income at 15%. It never applied the $20,200 standard deduction, the $6,000 senior deduction, the $1,000 non-itemizer charitable deduction or the 12% bracket."
-us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It treated 2026 as a post-TCJA-sunset year. It itemized miscellaneous employee expenses and uncapped SALT, subtracted a $5,300 personal exemption, and taxed ordinary income at 15% above $12,150. Under OBBBA the correct result comes from the $27,200 combined standard, senior and non-itemizer charitable deductions and 10%/12% brackets."
-us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It stopped at the $20,200 standard deduction and omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $50,446 is $7,000 too high, which also pushed $996 of qualified dividends into the 15% band."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied projected post-TCJA-expiration rules: $18,126 of itemized deductions, a $5,200 personal exemption and a 25% bracket. OBBBA kept the TCJA brackets and added the $6,000 senior and $1,000 non-itemizer charitable deductions on top of the $20,200 standard deduction, for $43,446.18 of taxable income."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It overstated AGI at $71,468 and reported $69,455 of taxable income after a $29,600 deduction, an arithmetic impossibility. It then also subtracted $10,808 of itemized deductions on top of the standard deduction, which is double-counting. It never built the correct $27,200 deduction stack (standard, senior and non-itemizer charitable), so its $4,658 does not come from a valid taxable income."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $6,300 implies about $54,500 of ordinary taxable income at 10%/12%, which fits deducting only a basic standard deduction and omitting the age/blind additions, the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Those deductions reduce taxable income to $43,446.18."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used TCJA-sunset rules: $17,023 of itemized deductions including $2,097 of 2%-floor employee business expenses, a $5,150 personal exemption, and pre-TCJA brackets. It also rounded the capital loss to $1,500. OBBBA made the standard deduction, the 10%/12% brackets and the suspension of miscellaneous deductions permanent, and it added the senior and non-itemizer charitable deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It computed about $4,917 of tax before credits, then cut it to $0 by citing nonrefundable credits. None applies: there are no dependents, and the elderly/disabled credit is fully phased out at $70,646 of AGI with $4,800 of nontaxable Social Security, so tax before refundable credits stays positive."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted only the basic plus age/blind standard deduction and omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Its $5,994 implies about $7,000 more taxable income than the correct $43,446.18."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It computed tax from itemized deductions instead of the larger $20,200 standard deduction plus the $6,000 senior and $1,000 non-itemizer charitable deductions, which overstated taxable income. It submitted $6,712 while its explanation ends with 6979, so the answer is inconsistent with itself as well."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used 2025 amounts ($15,000 base plus $2,000 each for age and blindness, $19,000 in total) instead of the 2026 $20,200. It also omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income was $51,646 and it taxed qualified dividends at 15% instead of 0%."
-us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of provisional income and AGI, which understated taxable Social Security at $26,289.75 when the correct figure is the $27,200 cap. It also used a $19,570 standard deduction and omitted the $6,000 senior and $1,000 non-itemizer charitable deductions, which overstated taxable income at $48,341.75."
-us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 0.85 times the excess over $34,000 ($23,340), dropping the $4,500 base amount. The statutory formula gives $27,840, which the $27,200 cap limits to $27,200. It also used $1,650 per age/blind addition instead of the single-filer $2,050. Both errors understated taxable income at $40,386, even though it correctly included the $6,000 senior and $1,000 charitable deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age/blind deductions fully offset taxable income. The full $27,200 of deductions (standard, senior and non-itemizer charitable) set against $70,646.18 of AGI still leaves $43,446.18 taxable, which produces $4,746.66 of tax."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It concluded that tax-exempt income and itemized deductions push tax to zero. Tax-exempt interest is not taxed and does not reduce tax, and $27,200 of deductions against $70,646.18 of AGI leaves $43,446.18 of taxable income taxed at 10%/12%."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-sunset' rules: about $17,023 of itemized deductions including 2%-floor miscellaneous expenses, a $5,300 personal exemption and pre-TCJA brackets. OBBBA made the TCJA standard deduction and 10%/12% brackets permanent and added the $6,000 senior and $1,000 non-itemizer charitable deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly stacked the enhanced standard deduction, the senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $43,546 is $100 above $43,446.18 because its basic-plus-age/blind standard deduction fell $100 short of $20,200 (16,100 + 2 × 2,050). That $100 difference produced the $12 overstatement."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its explanation names the standard, age/blind and senior deductions but not the $1,000 non-itemizer charitable deduction. Its $4,957 implies about $43,375 of ordinary taxable income, roughly $1,750 above the correct $41,622.18 after the full $27,200 deduction stack."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but omitted the $1,000 charitable deduction for single non-itemizers that starts in 2026. That left taxable income at about $44,465 instead of $43,446.18 and tax $122 too high."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed that taxable income after deductions falls below the filing threshold. With $70,646.18 of AGI and $27,200 of deductions, $43,446.18 remains taxable and produces $4,746.66 of tax, and the lack of earned income is irrelevant to that result."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected TCJA-sunset rules: itemized deductions including 2%-floor employee expenses and MN income tax, a $12,400 pre-TCJA standard deduction comparison, a $5,300 personal exemption and a 15% bracket. OBBBA made the $16,100-base standard deduction and 10%/12% brackets permanent and added the $6,000 senior and $1,000 non-itemizer charitable deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It itemized with miscellaneous employee expenses above the 2% floor, added a personal exemption and taxed income in a 15% bracket, all of which assume the TCJA expired. Under OBBBA the $27,200 standard, senior and non-itemizer charitable deductions and the 10%/12% brackets apply."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but omitted the $1,000 non-itemizer charitable deduction that starts in 2026. That left taxable income at $44,396 instead of $43,446.18. Small errors in the base ($16,150 vs $16,100) and the 10% bracket top ($12,250 vs $12,400) did not offset the $120 overstatement."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used 'post-TCJA reversion' rules: itemized deductions including 2%-floor miscellaneous expenses and state income tax, and 15% brackets indexed from 2017. OBBBA kept the TCJA 10%/12% brackets and gives $27,200 of standard, senior and non-itemizer charitable deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It itemized uncapped SALT with MN income tax and 2%-floor miscellaneous deductions, subtracted a $5,050 personal exemption and taxed at 15%, all of which assume the TCJA expired. OBBBA made the TCJA structure permanent, and the $27,200 of standard, senior and non-itemizer charitable deductions with 10%/12% brackets yields $4,746.66."
-us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
-us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It admitted taxable income is positive, then zeroed the tax with 'personal exemptions/credits' that do not exist in 2026. The personal exemption is permanently $0 under OBBBA, and no nonrefundable credit applies at $70,646 of AGI, so $43,446.18 of taxable income produces $4,746.66 of tax."
-us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its reasoning stacked the $20,200 standard deduction and the $6,000 senior deduction but omitted the $1,000 non-itemizer charitable deduction, which gave $44,446 of taxable income and $4,866.64 of tax. It then submitted $3,491.46, which contradicts its own computation."
-us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted only a $20,050 standard deduction and omitted the $6,000 senior and $1,000 non-itemizer charitable deductions. It also treated the $1,447 long-term capital loss as preferentially taxed gain. Its reasoning reached $5,544, but it submitted $9,505, which no step supports."
-us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left all Social Security out of AGI, although $27,200 is taxable, and applied an elderly/disabled credit that is fully phased out at this AGI and nontaxable Social Security. It also added a charitable 'deduction' to the tax instead of subtracting $1,000 from taxable income, so its $5,065 exceeds its own $3,933.60 tentative tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its deductions stopped at $26,200 (standard $20,200 plus the $6,000 senior deduction). It left out the 2026 $1,000 charitable deduction for non-itemizers, so it worked out $4,867 instead of $4,746.66. It then submitted $6,552, which contradicts its own computation."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted only 50% of Social Security ($16,000) as taxable. Provisional income of $61,459 makes the 85% cap of $27,200 taxable. It also made up a 'disabled' additional standard deduction, then cut its own $3,668 tentative tax to $1,065 by citing credits that don't exist."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the senior, blind and disability deductions wipe out all taxable income. In fact $70,646 of AGI less $27,200 of deductions leaves $43,446.18 taxable, which carries $4,746.66 of tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction. It left out the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $50,446 instead of $43,446.18 and pushed part of the qualified dividends into the 15% band."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used outdated $15,000/$2,000/$2,000 standard deduction amounts and missed the $1,000 non-itemizer charitable deduction, reaching about $5,011. It then cut that to $1,971 by counting the age/blind additional amounts a second time."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of about $43,546 was near the correct figure, but its bracket arithmetic failed. Tax on $41,722 of ordinary income is $1,240 + 12% × $29,322 ≈ $4,759, not $2,336, and it charged 15% on qualified dividends that fall entirely in the 0% band. It also used an inflated $17,000 base standard deduction and left out the $1,000 non-itemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and applied the old 10%/15%/25% brackets, a personal exemption, miscellaneous itemized deductions and an $11,400 standard deduction. OBBBA made the TCJA structure permanent, so 2026 uses the 10%/12% brackets, the $20,200 aged/blind standard deduction, the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $1,824 of qualified dividends as part of the $3,728 of non-qualified dividends, which understated AGI at $68,822. It also left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income came out at $48,622 instead of $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $20,200 standard deduction plus the $6,000 senior deduction. It left out the $1,000 charitable deduction that single non-itemizers can claim starting in 2026, so its taxable income was $1,000 too high ($44,446) and its tax $120 too high."
+us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used an outdated $19,200 standard deduction and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave taxable income of $51,446 instead of $43,446.18 and taxed part of the qualified dividends at 15%."
+us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a personal exemption, deductible miscellaneous employee expenses, an itemized state income tax deduction and a 15% second bracket. OBBBA made the TCJA brackets permanent and kept those items repealed, so the correct path is the $27,200 deduction stack and the 10%/12% brackets."
+us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It treated 2026 as a TCJA-sunset year, with a $5,300 personal exemption, 2%-floor miscellaneous deductions and 10%/15% brackets. Under OBBBA the correct path is the standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction, taxed at 10%/12%."
+us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave taxable income of $50,446 and pushed $996 of qualified dividends into the 15% band."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired, claiming $18,126 of itemized deductions plus a $5,200 personal exemption and taxing income in a 25% bracket. OBBBA kept the TCJA brackets and eliminated personal exemptions, so the correct path is the $27,200 deduction stack and 10%/12% rates."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its derivation doesn't hold together. It overstated AGI at $71,468, subtracted a made-up $29,600 standard deduction but still reported $69,455 of taxable income, and then also subtracted itemized deductions, which is not allowed alongside the standard deduction. The correct taxable income is $43,446.18 from the $27,200 stack of standard, senior and non-itemizer charitable deductions."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $6,300 implies roughly $50,000+ of taxable income, which matches leaving out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, or applying TCJA-sunset rules, instead of taxing $43,446.18 at 10%/12%."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules, with $2,097 of 2%-floor employee expenses, a $5,150 personal exemption and post-sunset brackets, and taxed qualified dividends at 15%. Under OBBBA, taxable income is $43,446.18 after the standard, senior and non-itemizer charitable deductions, and the qualified dividends are taxed at 0%."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It computed about $4,917 of tax and then reduced it to $0 by pointing to nonrefundable credits and deduction rules that don't apply. This filer has no available nonrefundable credit (the elderly/disabled credit is fully phased out at this AGI)."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It applied only the aged/blind standard deduction. It left out the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, and its $5,994 matches taxable income of about $51,600 instead of $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no numbers, and its submitted $6,712 contradicts the 'value = 6979' in its own explanation. Both figures come from itemizing instead of taking the $20,200 standard deduction plus the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used pre-OBBBA 2025 amounts ($15,000 + $2,000 + $2,000 = $19,000) and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave $51,646 of taxable income, and it then taxed the qualified dividends at 15% when they belong in the 0% band."
+us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It dropped the $1,824 of qualified dividends from provisional income and AGI, which understated taxable Social Security at $26,289.75 instead of the $27,200 cap. It also left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, and it used a $19,570 standard deduction instead of $20,200."
+us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 0.85 × (provisional income − $34,000) = $23,340, skipping the $4,500 add-on. The formula actually gives $27,840, which is capped at $27,200. It also used $3,300 for the age/blind amounts instead of 2 × $2,050 and a $12,200 10% bracket instead of $12,400, which understated tax at $4,383."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age/blind deductions fully offset income. AGI of $70,646.18 (including $27,200 of taxable Social Security) less $27,200 of deductions leaves $43,446.18 taxable and $4,746.66 of tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It wrongly treated tax-exempt income and implied itemized deductions as wiping out the liability. Tax-exempt interest only feeds the Social Security provisional-income test, itemized deductions ($14,573) are smaller than the standard deduction, and taxable income is $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-sunset' rules: $17,023 of itemized deductions including 2%-floor employee expenses, a $5,300 personal exemption and a 15% bracket. OBBBA made TCJA permanent, so the correct calculation uses the $27,200 standard/senior/charitable deduction stack and 10%/12% brackets."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly included the senior deduction and the $1,000 non-itemizer charitable deduction, but its total deductions were $27,100 instead of $27,200. It used $2,000 instead of $2,050 for each aged/blind additional amount, so taxable income came out at $43,546 and tax about $12 too high."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $4,957 implies about $1,750 less in deductions than the correct $27,200. It never mentioned the $1,000 non-itemizer charitable deduction ($120 of the gap) and understated the 2026 standard/aged/blind amounts for the rest."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but left out the $1,000 charitable deduction for non-itemizers that starts in 2026. That gave taxable income of about $44,465 instead of $43,446.18, roughly $120 too much tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It wrongly said income falls below filing thresholds because there is no earned income. Pension, IRA, dividend and 85% Social Security income produce $70,646 of AGI and $43,446.18 of taxable income."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used TCJA-sunset projections: a personal exemption, 2%-floor miscellaneous deductions, a state income tax itemized deduction and a 15% bracket. Under OBBBA, the standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction apply, and tax is computed at 10%/12%."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a personal exemption, 2%-floor miscellaneous employee expenses and a 15% ordinary bracket, which are TCJA-sunset rules. OBBBA made those changes permanent, so the $27,200 OBBBA deduction stack and the 10%/12% rates apply."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the standard, aged/blind and $6,000 senior deductions but left out the $1,000 charitable deduction for single non-itemizers that starts in 2026. That left taxable income at $44,396 instead of $43,446.18, and it also used slightly off $16,150 base and $12,250 bracket figures."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA reverted in 2026, itemizing state income tax and 2%-floor employee expenses and applying 10%/15% brackets inflated from 2017. OBBBA kept the TCJA structure, so taxable income is $43,446.18 after the standard, senior and non-itemizer charitable deductions, taxed at 10%/12%."
+us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a personal exemption, deductible miscellaneous expenses, state tax in SALT and a 15% bracket. Under OBBBA, the correct path is the $20,200 standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction, taxed at 10%/12%."
+us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It admitted taxable income was positive but claimed personal exemptions and credits reduce the tax to zero. Personal exemptions are $0 under OBBBA and no nonrefundable credit applies, so $43,446.18 of taxable income gives $4,746.66."
+us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its deductions stopped at $26,200 (standard plus senior) and left out the $1,000 charitable deduction for non-itemizers that starts in 2026, so its reasoning reached $4,866.64. It then submitted $3,491.46, which matches neither its own work nor the correct $4,746.66."
+us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. It also wrongly counted the $1,447 long-term capital loss as preferentially taxed income, and its submitted $9,505 contradicts its own computed $5,544."
+us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left Social Security out of AGI even though provisional income of $61,459 makes $27,200 taxable. It also applied an elderly/disabled credit that phases out completely above $17,500 of AGI, and its final $5,065 is larger than its own $3,933.60 tentative tax."
us,scenario_122,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 73, blind, and disabled as automatic categorical eligibility and conflated Medicare Savings Programs such as QMB and SLMB with full Medicaid eligibility. It also dismissed resource limits despite the disclosed $44,000 in bank assets and $177,000 in stock assets; the head qualified through none of Minnesota’s Medicaid pathways."
us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found income far above the 138% FPL MAGI limit, then reversed that result solely because the head was elderly, blind, and disabled. Those characteristics do not automatically confer Medicaid eligibility, and the head qualified through no alternative category."
us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base."
us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract."
us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It treated Minnesota's Social Security subtraction as phased out at this income, but $70,646 AGI is below the single-filer threshold, so the full $27,200 is subtracted. It also claimed itemized deductions beat the standard deduction, though charitable plus property tax ($14,926) is less than the $19,300 aged/blind standard deduction. That kept about $52,000 taxable and pushed income into the 6.8% bracket instead of $19,169 taxed at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The structure was right, but two deduction amounts were too high. It used a $19,500 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $18,792 instead of $19,169, so tax was $1,005.37 instead of $1,025.54."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Its own numbers leave about $30,000 of Minnesota taxable income: $43,446 of non-Social Security income minus a $13,175 standard deduction. It still reported $0 with no deduction or credit to support it. It also understated the aged/blind standard deduction ($19,300) and missed the $4,977.18 nonitemizer charitable subtraction, which together leave $19,169 taxed at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,age_disability,False,"It correctly reached about $19,442 of taxable income and about $1,040 of tax. It then zeroed the tax by applying a Minnesota senior/elderly-disabled subtraction and a long-term-care credit. The elderly/disabled subtraction phases out entirely at $70,646 AGI, and the facts support no long-term-care credit. The tax is $19,169 × 5.35% = $1,025.54."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It never computed taxable income. It asserted that the Social Security subtraction plus deductions bring income near zero, and it wrongly treated the standard deduction as a nonrefundable credit. Subtracting $27,200 of Social Security, the $19,300 standard deduction and the $4,977.18 charitable subtraction from $70,646.18 AGI still leaves $19,169 taxed at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated Minnesota's Social Security subtraction as only partial because of income limits. At $70,646 AGI the full $27,200 is subtracted. It also skipped the $4,977.18 nonitemizer charitable subtraction and understated the aged/blind standard deduction. Its $1,650 implies about $30,800 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $19,300 standard deduction and the full $27,200 Social Security subtraction. But it computed the charitable subtraction as 50% × ($10,808 − $500) = $5,154, while PolicyEngine's subtraction is $4,977.18. Taxable income came out at $18,992 instead of $19,169, which accounts for the $9 shortfall."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It invented a $15,000 Minnesota subtraction for private pension and IRA income at age 65+. Minnesota has no such subtraction for this income, and the error removed $15,000 that is actually taxable. It also used a $14,575 standard deduction without the age and blind additions, so it chose itemizing at $18,436. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It left taxable Social Security out of federal AGI and then described the Social Security subtraction as shrinking income further, so it double-counted the exclusion. It also invoked a Minnesota disability/elderly credit to reach $0. The elderly/disabled subtraction is zero at this AGI, and $43,446 minus the $19,300 standard deduction and the $4,977.18 charitable subtraction leaves $19,169 taxed at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It correctly subtracted all $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300. That left $24,046 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It skipped the full $27,200 Minnesota Social Security subtraction. It also subtracted a personal exemption of about $5,300, which Minnesota does not give filers, and used a standard deduction without the age and blind additions. That produced $50,146 of taxable income reaching the 6.8% bracket instead of $19,169 at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction and invented a $5,094 personal exemption. It also used $17,023 of itemized deductions, which is less than the $19,300 aged/blind standard deduction. Taxable income came out at $48,529 in the 6.8% bracket instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It correctly subtracted $27,200 of Social Security. It then used an $18,500 standard deduction instead of $19,300 and left out the $4,977.18 nonitemizer charitable subtraction. That left $24,946 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted an invented $5,000 retirement subtraction, which does not exist for private pension and IRA income. It also used an $18,500 standard deduction instead of $19,300 and left out the $4,977.18 charitable subtraction. Its $19,946 of taxable income is close only because these errors partly offset."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied only the standard deduction to the full $70,646 AGI. It skipped the $27,200 Social Security subtraction and the $4,977.18 charitable subtraction. It also used 7.05% as the second-bracket rate, but Minnesota's second rate is 6.8%. The result was $51,946 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $2,420 implies about $42,500 of taxable income reaching the 6.8% bracket. That means it did not combine the full $27,200 Social Security subtraction, the $19,300 aged/blind standard deduction and the $4,977.18 charitable subtraction, which together leave $19,169 at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied only a partial Social Security subtraction. With $70,646 AGI, below the single-filer phaseout, Minnesota subtracts the full $27,200. Keeping that income taxable pushed it into the 6.8% bracket, and it also left out the $4,977.18 charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It called its Social Security subtraction partial, but its $51,665 of taxable income reflects essentially no subtraction from the roughly $70,665 AGI. The full $27,200 should have been subtracted. It also left out the $4,977.18 charitable subtraction, taxing income at 6.8% instead of $19,169 at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,111 implies about $20,770 of taxable income, roughly $1,600 above the correct $19,169. That means it under-applied the combined $19,300 standard deduction and $4,977.18 charitable subtraction after the $27,200 Social Security subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly subtracted $27,200 of Social Security. It then used an $18,270 standard deduction instead of $19,300 and left out the $4,977.18 nonitemizer charitable subtraction. That left $25,176 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It submitted $2,389, but its explanation ends with value = 1308, so the answer contradicts itself. The $1,308 figure matches $24,446 × 5.35%: it subtracted Social Security but used a $19,000 standard deduction and left out the $4,977.18 charitable subtraction. Neither number reflects the correct $19,169 × 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It correctly subtracted the full $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,000 standard deduction instead of $19,300. That left $24,446 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied rates of 2.85% and 4.80%, which are not Minnesota rates; Minnesota's first bracket is 5.35%. It used a $14,600 standard deduction without the age and blind additions, so it chose $14,926 of itemized deductions over the correct $19,300 standard deduction. It also left out the $4,977.18 charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used an $18,560 standard deduction instead of $19,300. It also left out the $4,977.18 nonitemizer charitable subtraction. That left $24,886 taxable instead of $19,169. Its understated taxable Social Security ($23,340) canceled out through the subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It assumed senior/disabled subtractions wipe out Minnesota taxable income. The elderly/disabled subtraction phases out to zero at $70,646 AGI. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It asserted that tax after nonrefundable credits is zero without computing taxable income. Minnesota taxable income is $70,646.18 − $27,200 − $19,300 − $4,977.18 = $19,169, which produces $1,025.54 at 5.35%, and no nonrefundable credit applies."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine's calculation does not include. That raised taxable income to about $20,450 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It said it applied the Social Security subtraction, but its $49,900 of taxable income equals AGI minus only a standard deduction, so the $27,200 subtraction was never taken. It also left out the $4,977.18 charitable subtraction and taxed income in the 6.8% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine does not include. It also used a $5,154 charitable subtraction instead of $4,977.18. That raised taxable income to $21,055 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $2,640 implies about $45,500 of taxable income reaching the 6.8% bracket. That matches AGI minus the standard and charitable deductions with no Social Security subtraction. The full $27,200 subtraction applies at this AGI."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The structure was right, but it used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $18,892 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied the Social Security subtraction and an aged/blind standard deduction, but it left out the $4,977.18 nonitemizer charitable subtraction. That left $23,265 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 of taxable income means it overstated its deductions by about $827. It computed the charitable subtraction as $5,154 instead of $4,977.18, and its standard deduction was roughly $650 above $19,300."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted $27,200 of Social Security and a $19,400 standard deduction, but it left out the $4,977.18 nonitemizer charitable subtraction. It also used $19,400 instead of the $19,300 standard deduction, leaving $24,046 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,age_disability,False,"It claimed age-based exemptions zero out Minnesota taxable income. Minnesota gives filers no personal exemption, and the elderly/disabled subtraction is zero at $70,646 AGI. $19,169 remains taxable at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It correctly subtracted all $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,600 standard deduction instead of $19,300. That left $23,844 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Its $22,000 of taxable income means it did not combine the $19,300 aged/blind standard deduction with the separate $4,977.18 nonitemizer charitable subtraction after subtracting Social Security. Those together leave $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine does not include. It also used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $20,905 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction. It also itemized $17,023, which is below the $19,300 aged/blind standard deduction, and left out the charitable subtraction. That taxed $53,623 across the 5.35% and 6.8% brackets instead of $19,169 at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It started near the full $70,600 AGI and reached about $50,000 of taxable income. So it never took the $27,200 Social Security subtraction or the $4,977.18 charitable subtraction and wrongly taxed income at 6.8%."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It subtracted a $5,330 personal exemption, which Minnesota does not give filers, and left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,425 standard deduction. Its $18,690 of taxable income lands near $19,169 only because the invented exemption offsets the missing charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It applied a senior age-65 subtraction to reach zero taxable income. That subtraction phases out to nothing at $70,646 AGI. The Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction leave $19,169 taxable at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a $15,300 standard deduction without the $4,000 of age and blind additions, so it chose $17,367 of itemized deductions. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction. That left $26,079 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction and added back tax-exempt interest and tax-exempt pension income. It also used a $14,600 standard deduction without the age and blind additions. It then cut the tax with the property tax refund, which is a separate refund program and not a nonrefundable credit against income tax."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It misstated federal AGI as $52,092 instead of $70,646 and used a made-up $30,100 standard deduction instead of $19,300. It also left out the Social Security and charitable subtractions. Its reported $1,450 does not even equal 5.35% of its own $21,992 taxable income."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It said the Minnesota Social Security subtraction was phased out at this income. In fact AGI of $70,646 is below the single-filer phase-out threshold, so the full $27,200 is subtracted. Without that subtraction it got about $52,000 of taxable income in the 6.8% bracket instead of $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The method was right (AGI minus the $27,200 Social Security subtraction, standard deduction and charitable subtraction, taxed at 5.35%), but two inputs were too high. It used a $19,500 standard deduction instead of $19,300, and a charitable subtraction of $5,154 (50% × ($10,808 − $500)) instead of the engine's $4,977.18. That gave $18,792 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $27,200 of taxable Social Security out of the income base and then declared the tax zero without doing the math. Even its own figures ($43,446 of income minus a $13,175 deduction) leave about $30,000 taxable. The correct path leaves $19,169 taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed a pension subtraction that does not apply to this filer and assumed the deductions wipe out all tax. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It got close to the right path (about $19,442 taxable and about $1,040 of tax). It then wrongly applied Minnesota's elderly/disabled subtraction, which is fully phased out for this filer because AGI and nontaxable Social Security exceed the limits. It also applied a long-term care credit even though there are no long-term care premiums. Together these cut the tax to $0 instead of $1,025.54."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It asserted that the Social Security subtraction plus deductions push taxable income to near zero, and it made up nonrefundable credits for age and blindness. Subtracting $27,200 of Social Security, the $19,300 standard deduction and the $4,977.18 charitable subtraction from $70,646 still leaves $19,169 taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security subtraction as only partial, when the full $27,200 applies below the phase-out threshold. It also understated the age/blind add-ons (about $3,300 instead of the amount that brings the deduction to $19,300) and left out the non-itemizer charitable subtraction. Those errors produced about $44,000 of taxable income spanning two brackets."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The Social Security subtraction ($27,200) and the standard deduction ($19,300) were both correct. The only error was the charitable subtraction: it used 50% × ($10,808 − $500) = $5,154 instead of the engine's $4,977.18. That understated taxable income by $176.82 ($18,992 vs. $19,169)."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income, even though Minnesota has started from federal AGI since 2019. It made up a $15,000 Minnesota pension/IRA subtraction that does not exist for this filer. It also itemized with unreimbursed employee expenses instead of taking the $19,300 age/blind standard deduction plus the charitable subtraction. The result was $10,010 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left taxable Social Security out of federal AGI ($43,446) and then still talked about subtracting Social Security again. It also made up an elderly/disability credit to reach zero. In fact, $19,169 of taxable income remains after the $27,200 subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the full $27,200 of Social Security but left out the $4,977.18 non-itemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300, which gave $24,046 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It skipped the full $27,200 Social Security subtraction. It also subtracted a $5,300 personal exemption, which Minnesota no longer allows for the filer (only dependents get exemptions), and used a $15,200 standard deduction with no age/blind add-ons. The resulting $50,146 of taxable income fell partly in the 6.8% bracket."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted a personal exemption that Minnesota does not allow for the filer and $17,023 of itemized deductions. It never took the $27,200 Social Security subtraction and never compared against the $19,300 age/blind standard deduction plus the charitable subtraction. That gave $48,529 of taxable income in two brackets instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but understated the age/blind standard deduction ($18,500 vs. $19,300). It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,946 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It made up a $5,000 retirement subtraction, used an $18,500 standard deduction instead of $19,300, and left out the $4,977.18 charitable subtraction. The errors partly cancel, landing at $19,946 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It never subtracted the $27,200 of taxable Social Security, which Minnesota excludes in full at this AGI, and it left out the charitable subtraction. It also taxed the excess at 7.05%, but Minnesota's second bracket rate is 6.8%. The result was $51,946 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"A $2,420 tax implies about $42,500 of Minnesota taxable income, spanning the 5.35% and 6.8% brackets. That is consistent with not taking the full $27,200 Social Security subtraction. The correct derivation leaves $19,169 taxed entirely at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied only a partial Social Security subtraction. In fact AGI of $70,646 is below the single-filer phase-out threshold, so the full $27,200 is subtracted. The leftover income pushed taxable income into the 6.8% bracket instead of leaving $19,169 at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied only a partial Social Security subtraction even though the full $27,200 is subtracted at this AGI, and it left out the charitable subtraction. That gave $51,665 of taxable income in two brackets instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"A $1,111 tax implies about $20,766 of taxable income at 5.35%, roughly $1,600 above the correct $19,169. That is consistent with taking the Social Security subtraction but understating the combined $19,300 age/blind standard deduction and $4,977.18 charitable subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but used an $18,270 standard deduction instead of $19,300. It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $25,176 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $2,389, which contradicts its own explanation's figure of $1,308. Neither number reflects the correct path: $70,646 AGI minus the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction. The $2,389 figure implies taxable income in the 6.8% bracket, consistent with an incomplete Social Security subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the full $27,200 of Social Security but used a $19,000 standard deduction instead of $19,300. It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,446 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used made-up rates of 2.85% and 4.80%; Minnesota's lowest rate is 5.35%. It also chose $14,926 of itemized deductions over a $14,600 standard deduction that left out the age and blindness add-ons. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction, giving $19,169 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It reached the correct $43,446 Minnesota base after the Social Security subtraction. However, it used an $18,560 standard deduction instead of $19,300 and left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,886 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed senior/disabled subtractions wipe out taxable income. In fact the elderly/disabled subtraction is fully phased out at this income. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave a zero tax with no computation. The correct path leaves $19,169 of Minnesota taxable income after the Social Security subtraction, the age/blind standard deduction and the charitable subtraction, producing $1,025.54 at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It added back the $2,013 of tax-exempt interest as a Minnesota addition. The facts do not identify it as non-Minnesota bond interest, and the engine applies no such addition. Adding it raised taxable income to about $20,450 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"About $49,900 of taxable income means it did not apply the full $27,200 Social Security subtraction or the $4,977.18 charitable subtraction. Both apply in full at this AGI, and the correct base is $19,169 taxed entirely at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It added the $2,013 of tax-exempt interest back as a Minnesota addition, which the engine does not do for these facts. That raised taxable income to $21,055. Without the addition, its subtractions land near the correct $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"A $2,640 tax implies about $45,800 of taxable income reaching the 6.8% bracket, consistent with not taking the full $27,200 Social Security subtraction. It also left out the charitable subtraction. The correct base is $19,169 taxed entirely at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It implicitly used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of the engine's $4,977.18. That understated taxable income at $18,892 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took the Social Security subtraction and the age/blind standard deduction but left out the $4,977.18 non-itemizer charitable subtraction. That left $23,265 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 of taxable income implies about $25,100 of combined standard deduction and charitable subtraction. The correct total is $24,277.18 ($19,300 standard deduction + $4,977.18 charitable subtraction), so it overstated the deductions and understated taxable income relative to $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but left out the $4,977.18 non-itemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300, leaving $24,046 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed age-based exemptions bring taxable income to zero. Minnesota has no such exemption at this income level. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 is taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $19,600 standard deduction instead of $19,300 and left out the $4,977.18 non-itemizer charitable subtraction. That gave about $23,844 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"After the Social Security subtraction and the standard deduction, it did not apply the $4,977.18 non-itemizer charitable subtraction. It ended up with about $22,000 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It added back the $2,013 of tax-exempt interest, which the engine does not treat as a Minnesota addition for these facts. It also used a $19,400 standard deduction and a $5,154 charitable subtraction instead of $19,300 and $4,977.18, giving $20,905 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It never took the $27,200 Social Security subtraction. It itemized $17,023 instead of taking the larger $19,300 age/blind standard deduction plus the $4,977.18 charitable subtraction. That produced $53,623 of taxable income in the 6.8% bracket."
+us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It kept taxable income near $50,000 by not taking the full $27,200 Social Security subtraction, which applies in full below the phase-out threshold. It also left out the charitable subtraction, putting income in the 6.8% bracket instead of leaving $19,169 at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,025.54 result."
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $5,330 personal exemption, but Minnesota allows exemptions only for dependents, not the filer. It also left out the $4,977.18 non-itemizer charitable subtraction. Those errors largely cancel, giving $18,690 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It relied on a senior age-65 subtraction that is fully phased out at this income and concluded taxable income is zero. In fact $19,169 remains after the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It itemized $17,367 against a $15,300 standard deduction that left out the age and blindness add-ons. The correct $19,300 standard deduction plus the $4,977.18 non-itemizer charitable subtraction totals $24,277 and wins. That left $26,079 taxable instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took no Social Security subtraction and added back tax-exempt interest and pension income. It used a $14,600 standard deduction with no age/blind add-ons. It then cut the liability with the property tax refund, which is a separate refund and not a nonrefundable income tax credit."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used the wrong federal AGI ($52,092 instead of $70,646) and an invented $30,100 standard deduction, and it never took the Social Security or charitable subtractions. It also made an arithmetic error: $21,992 at 5.35% is $1,177, not $1,450."
us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed household income correctly at about $77,500 and even said that was near the income limit. It still applied a 2.4% threshold with a 50% state copayment to $4,800 of rent-as-tax and got about $1,470. It then raised that to $2,265 while saying the phaseout reduces the refund. At this household income the 2026 MN renter's credit is $0, so no refundable state credit applies."
us,scenario_122,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It found the right household income ($77,459) and rent-as-tax ($4,800). It then applied a mid-schedule bracket: a 2.5% income threshold with a 50% state share, and no maximum-refund cap. The MN renter's credit schedule is fully phased out at this income, so the credit is $0, not $1,431.74."
us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It assumed the head qualifies for the Minnesota renter's refund and simply entered a $2,000 'maximum allowable refund'. It never checked the household income of about $77,460 against the renter's credit income schedule. That schedule pays nothing at this income, so state refundable credits are $0."
@@ -9821,51 +10059,52 @@ us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missin
us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the 2026 OBBBA charitable deduction for non-itemizers ($2,000 on a joint return). It therefore taxed $67,802 instead of $65,802 and got $5,440 for the parents instead of $5,200.24. It then added the child's separate tax and submitted $5,860, which matches neither its $5,440 parent-only result nor its $8,660 combined total."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It put the dependent child's $45,000 of wages into joint AGI ($145,002), but the tax unit's AGI counts only the head's and spouse's income ($100,002). Its deductions ($32,200 plus $2,000), 2026 brackets and $2,200 CTC were right. The extra $45,000 of taxable income ($110,802 vs $65,802) explains the whole $6,400.20 overstatement."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the outdated $29,200 standard deduction and invented $25,200 of mortgage interest from the $420,000 balance, although no interest paid is listed, so mortgage interest is $0. It also claimed a $4,000 CTC for a second child who does not exist. The correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, $65,802 of taxable income and one $2,200 CTC; its $8,316 does not follow from any of its own steps."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $31,500 instead of the 2026 amount of $32,200, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of the 2026 amount of $2,200, reaching $5,724. It then submitted $5,316 after an unexplained 'adjustment' that its computation does not support."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It invented $29,400 of mortgage interest from the balance alone, although no interest paid is listed. It also dropped the $2,200 CTC by wrongly deciding the 16-year-old fails the support test, when the child is the couple's qualifying child. Its $12,053 matches neither its own ~$5,765 figure nor the correct path: $32,200 standard plus $2,000 charitable deduction, $7,400.24 of tax, minus the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $2,000 charitable deduction that non-itemizing joint filers get in 2026, so it taxed $67,802 instead of $65,802. It also mis-added the bracket tax ($7,610 instead of $7,640.24 on its own base) and ended at $5,410 instead of $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002), but the parents' tax unit AGI is only $100,002. With the right $32,200 standard deduction, $2,000 charitable deduction and $2,200 CTC, that $45,000 of extra taxable income is the entire gap between its $11,600.44 and $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated 2026 figures by inflating 2025 values: a $30,750 standard deduction, a 10% bracket up to $24,450 and a $2,000 CTC. The enacted OBBBA figures are $32,200, $24,800 and $2,200. It also left out the $2,000 charitable deduction, producing $5,821 instead of $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It stated that $18,557 of itemized deductions beat the ~$32,200 standard deduction it had just cited and itemized. The correct deduction is the $32,200 standard deduction plus the $2,000 charitable deduction ($34,200 total). It then submitted $15,300, which contradicts its own estimate of $7,300."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It gave the head a $13,650 overtime deduction, but only Child 1 is flagged as paid hourly, so the head has no qualifying overtime pay and gets no overtime deduction. It also added the child's $3,220 separate tax, even though the output is the joint unit's liability alone ($7,400.24 minus the $2,200 CTC = $5,200.24)."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002 instead of $100,002). It also used a $30,800 standard deduction instead of $32,200, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of $2,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the expired pre-TCJA rules: personal exemptions, 10%/15% brackets and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction plus a $2,000 charitable deduction, no exemptions, 10%/12% brackets and a $2,200 CTC. It also added $4,979.25 of separate tax for the child, although the output covers only the joint unit."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three $5,300 personal exemptions, a 15% bracket and a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's $4,896.25 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules: itemized deductions plus $15,900 of personal exemptions, 10%/15% brackets and a $1,000 CTC. Under OBBBA, 2026 has the $32,200 standard deduction, the $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's $4,855 separate tax, which is not part of the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (personal exemptions, a $1,000 CTC) and invented $16,800 of mortgage interest from a balance with no interest paid listed, so it itemized. Correct 2026 law gives the $32,200 standard deduction plus the $2,000 charitable deduction and a $2,200 CTC. It also added the child's $4,955 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It taxed the joint unit on $145,002 of AGI, counting the child's $45,000 of wages that are not the parents' income. Its $17,290 is higher even than the $13,800.44 of tax before credits that 2026 brackets give on that inflated base, so it also understated the $34,200 of deductions (standard plus charitable) and never subtracted the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (itemizing plus personal exemptions, restored brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's ~$4,925 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three personal exemptions totaling $15,150, 10%/15% brackets, a $1,000 CTC). OBBBA kept the TCJA structure for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's $5,023 separate tax, although the output is the joint unit's liability alone."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. The correct path is $100,002 of AGI minus the $32,200 standard deduction and $2,000 charitable deduction, $7,400.24 of tax on $65,802, minus the $2,200 CTC. Its $5,304 matches that same path with 2025 figures (a $31,500 standard deduction and a 10% bracket ending at $23,850 give $5,303.24), so it used last year's amounts instead of 2026's $32,200 and $24,800."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI, used expired pre-TCJA rules (personal exemptions, restored brackets) and treated the CTC as phased out. Under 2026 OBBBA law the parents' AGI is $100,002, they get the $32,200 standard deduction, and their $2,200 CTC phases out only above $400,000 for joint filers."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI and used expired pre-TCJA rules (itemizing $18,557 plus personal exemptions, restored brackets). The parents' AGI is $100,002, the OBBBA $32,200 standard deduction plus $2,000 charitable deduction applies, and the $2,200 CTC is fully used."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002). It also used a $30,000 standard deduction and 2025 bracket thresholds instead of 2026's $32,200 and $24,800/$100,800, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of $2,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It added the child's $3,312.08 separate tax to the parents' liability, although the output is the joint unit's figure alone. For the parents it also used an inflated $30,750 standard deduction and estimated brackets instead of 2026's $32,200 and a 10% bracket up to $24,800. It left out the $2,000 charitable deduction and used a $2,000 CTC instead of $2,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It put the dependent's $45,000 of wages into joint AGI and subtracted a second $16,100 'dependent standard deduction' that does not exist on a joint return. It then withheld the nonrefundable CTC by treating a $2,400 credit as fully refundable. In fact the $2,200 CTC first offsets liability, so all of it is used against the $7,400.24 of tax on the parents' $65,802 of taxable income."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It combined the child's $45,000 with the head's wages into a $145,002 joint AGI instead of the parents' $100,002. Its $7,313 also does not follow from its own stated taxable income of ~$116,751, on which 2026 brackets give over $15,000 of tax before credits."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It gave no computation. The correct figure is $7,400.24 of tax on $65,802 of taxable income ($100,002 AGI minus the $32,200 standard and $2,000 charitable deductions), minus the $2,200 CTC. Its $14,677 is about the size of the tax before credits on a $145,002 AGI that includes the child's $45,000 of wages, so it counted the dependent's wages and did not subtract the CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002) and left out the $2,000 charitable deduction, taxing $112,802 instead of $65,802. Its standard deduction, brackets and $2,200 CTC were correct."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002), but the parents' tax unit AGI is $100,002. Its $32,200 standard deduction, $2,000 charitable deduction, 2026 brackets and $2,200 CTC were all correct, so that $45,000 alone raises the answer from $5,200.24 to $11,600.44."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Its $15,128.44 of tax before credits equals $145,002 of AGI minus a $30,000 standard deduction under 2025 brackets. So it counted the child's $45,000 of wages, used a pre-OBBBA standard deduction and last year's thresholds instead of $32,200 and 2026 brackets, and left out the $2,000 charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($5,200.24) and then added $3,220 from a separate return for the dependent child. The output is the joint unit's liability, which does not include the child's own return, so the answer stays at $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,household_unit_or_filing_status,False,"It added the child's $3,220 separate tax to the joint unit's liability, which is not part of this output. It also left out the $2,000 charitable deduction, taxing $67,802 instead of $65,802 and getting $5,440.24 instead of $5,200.24 for the parents."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It took a $13,650 overtime deduction for the head, but only Child 1 is flagged as paid hourly, so the head gets no overtime deduction. It also left out the $2,000 charitable deduction and added the child's $3,220 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($5,200.24) and then added $3,220 from a separate return for the dependent child. The requested output is the joint unit's liability alone, so the child's own return tax does not belong in it."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002) and deducted mortgage interest, although only the loan balance is given and interest paid is $0. The correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, $7,400.24 of tax and a $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (itemizing $21,627 plus three personal exemptions, restored brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's ~$4,851 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into a $145,002 joint AGI, used expired pre-TCJA rules (exemptions, 15%/25% brackets) and phased out the CTC above an old $110,000 threshold. Under 2026 OBBBA law the parents' AGI is $100,002 and they get the $32,200 standard deduction and 10%/12% brackets. Their $2,200 CTC is safe because it phases out only above $400,000 for joint filers."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI, used expired pre-TCJA rules (three $5,400 exemptions, 15%/25% brackets) and phased out the CTC at an old $110,000 threshold. The parents' $100,002 AGI gets the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 10%/12% brackets and a full $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three $5,434 exemptions, 10%/15% brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's $4,846 separate tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It added the child's separate $3,225 tax to the parents' ~$5,210, although the output is the joint unit's liability alone. Its parent figure was also mis-added: the tax on $65,802 is $7,400.24, not $7,410, so the correct result is $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for this output, so there was nothing to compare with the $5,200.24 that comes from $65,802 of taxable income minus the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and made up a $12,000 overtime deduction (12% of wages) for a head who is not flagged as paid hourly and so gets no overtime deduction. It also left out the $2,000 charitable deduction. The correct base is $100,002 of AGI minus $34,200 of deductions."
-us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It claimed credits wipe out the whole liability, but tax before credits is $7,400.24 on $65,802 of taxable income, far more than the single $2,200 CTC. Refundable credits are excluded from this output by definition. The correct result is $5,200.24, not $0."
-us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and took a $12,500 overtime deduction for a head who is not flagged as paid hourly, so the head gets no overtime deduction. It also left out the $2,000 charitable deduction. The correct base is the parents' $100,002 AGI minus $34,200 of deductions."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and invented $27,300 of mortgage interest from a balance with no interest paid listed, so it itemized. It also used 2025 brackets and a $2,000 CTC. The correct path is the parents' $100,002 AGI, the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 brackets and a $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It invented items the facts do not support: a QBI deduction on W-2 wages, $16,800 of mortgage interest from a balance with no interest listed, a CDCC with no care expenses, and a foreign earned income exclusion. It then zeroed out a liability that its own arithmetic put at $5,687. With only the $32,200 standard deduction, the $2,000 charitable deduction and the $2,200 CTC, the tax is $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It left out the 2026 OBBBA $2,000 MFJ non-itemizer charitable deduction, so it taxed $67,802 instead of $65,802. It also added the child's separate $3,220 tax to the parents' tax. The $5,860 it submitted does not match either its $5,440 parents-only figure or its $8,660 combined figure."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put Child 1's $45,000 of wages into the parents' joint AGI ($145,002), but the child's wages belong on the child's own return. The correct AGI is $100,002 and taxable income is $65,802. Its deduction, bracket and CTC steps were otherwise right."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It made up about $25,200 of mortgage interest from the loan balance and chose to itemize, but no interest paid was listed, so it counts as $0. It also used a $29,200 standard deduction, counted two children for the CTC, and submitted $8,316 without a supporting calculation. The correct path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, then one $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,000 of combined wages, including the child's $45,000, on the joint return. It also left out the $2,000 non-itemizer charitable deduction and the $2,200 CTC. Correctly, the parents have $65,802 of taxable income and $7,400.24 of tax, reduced by the $2,200 credit."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used the 2025 standard deduction of $31,500 instead of the 2026 figure of $32,200, and a $2,000 CTC instead of the OBBBA $2,200. It also left out the $2,000 non-itemizer charitable deduction. It then submitted $5,316, which does not match its own $5,724 result."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $29,400 of mortgage interest by applying 7% to the $420,000 balance, but no interest paid was listed, so it counts as $0. It also wrongly denied the CTC on support-test grounds, and its final $12,053 has no supporting calculation. Correctly, the standard deduction plus the $2,000 charitable deduction applies, and the $2,200 CTC is allowed."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It used the correct $100,002 AGI and $32,200 standard deduction but left out the 2026 $2,000 non-itemizer charitable deduction, so it taxed $67,802 instead of $65,802. On top of that, it understated tax on its own base ($7,610 instead of $7,640.24)."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002), when they belong on the child's own return. With AGI of $100,002, taxable income is $65,802 and tax is $7,400.24 before the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It estimated the 2026 parameters by indexing (a $30,750 standard deduction and a 10% bracket ending at $24,450), when OBBBA sets them at $32,200 and $24,800. It also used a $2,000 CTC instead of $2,200 and left out the $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted $18,557 of itemized deductions after stating they were less than the $32,200 standard deduction, so it should have taken the standard deduction plus the $2,000 non-itemizer charitable deduction. Its submitted $15,300 also contradicts its own $7,300 estimate."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for Head, but Head is not flagged as paid hourly, so there is no qualifying overtime premium. It also added the child's separate $3,220 tax. Without those two errors, its parents' figure would have been $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into the joint AGI ($145,002). It also used a $30,800 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired for 2026 and applied personal exemptions, 15% brackets and a $1,000 CTC, when OBBBA made the TCJA structure permanent with a $32,200 standard deduction and a $2,200 CTC. It also added the child's separate-return tax to the parents' liability."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA rules for 2026 (personal exemptions of $5,300, itemizing, and a $1,000 CTC), but OBBBA keeps the $32,200 standard deduction, no exemptions, and a $2,200 CTC. It also added the child's $4,896.25 separate tax to the parents' figure."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired: it applied $15,900 of personal exemptions, itemized, used the 15% bracket, and gave a $1,000 CTC. It also added the child's separate $4,855 tax. Under 2026 OBBBA law, the parents alone have $65,802 of taxable income and owe $5,200.24 after the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired, made up $16,800 of mortgage interest even though no interest paid was listed, and claimed personal exemptions with a $1,000 CTC. It then added the child's separate $4,955 tax. The correct path is the 2026 standard deduction plus the $2,000 charitable deduction, the 10%/12% brackets, and a $2,200 CTC for the parents only."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It started from a joint AGI of $145,002 that includes the child's $45,000 of wages, and it gave no bracket or credit calculation to support $17,290. The parents' AGI is $100,002, and the $2,200 CTC reduces their $7,400.24 of tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA 2026 rules (personal exemptions, itemizing, and a $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added the child's separate $4,925 tax to the joint return's liability."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired, so it itemized $18,557, claimed $15,150 of exemptions, and applied a $1,000 CTC. It also added the child's $5,023 separate tax. OBBBA instead gives a $32,200 standard deduction, a $2,000 non-itemizer charitable deduction and a $2,200 CTC, applied to the parents' return only."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. The correct calculation is $100,002 AGI minus $32,200 and $2,000 equals $65,802 of taxable income, $7,400.24 of tax at 10%/12%, and $5,200.24 after the $2,200 CTC. Its $5,304 is $103.76 too high, which fits approximate brackets or deductions rather than the exact 2026 OBBBA figures."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in a $145,002 joint AGI and applied pre-TCJA personal exemptions. It also treated the CTC as phased out, but the 2026 phaseout starts at $400,000 MFJ, so the full $2,200 credit applies."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used a $145,002 joint AGI that includes the child's wages, then itemized $18,557 and claimed pre-TCJA personal exemptions. Under 2026 law the parents' AGI is $100,002, the $32,200 standard deduction plus the $2,000 charitable deduction applies, and the $2,200 CTC is subtracted."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI of $145,002. It also used the 2025 $30,000 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's separate $3,312 tax to the parents' liability, but the output covers only the joint return. On the parents' side, it estimated a $30,750 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's wages into the joint AGI and subtracted a second $16,100 standard deduction for the child on the parents' return. It also claimed the CTC was fully refundable, but the CTC first offsets tax as a nonrefundable credit. Because the $7,400.24 liability is larger than the credit, the whole $2,200 is subtracted here."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It combined Head's and the child's wages into a $145,000+ joint income, so its taxable income was about $116,751 instead of $65,802. The $7,313 it submitted also does not follow from its own taxable income figure."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. Its $14,677 fits taxing the child's $45,000 of wages on the joint return without the full deductions and CTC. Correctly, the parents' $100,002 AGI leaves $65,802 of taxable income and $5,200.24 of tax after the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the child's $45,000 of wages as part of the joint AGI ($145,002), and it also left out the 2026 $2,000 non-itemizer charitable deduction, so it taxed $112,802 instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002), when they are taxed on the child's own return. Its deductions and $2,200 CTC were otherwise correct, so it overstated taxable income by $45,000."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the child's wages on the joint return and used the outdated $30,000 standard deduction instead of the 2026 $32,200. It also left out the $2,000 non-itemizer charitable deduction, which produced $15,128.44 of regular tax instead of $7,400.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the parents' joint tax as $5,200.24, then added the child's separate $3,220 tax. That is wrong because this output covers only the parents' tax unit."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's separate $3,220 tax to the parents' liability. It also left out the $2,000 non-itemizer charitable deduction, so the parents' taxable income was $67,802 instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for Head, but Head is not flagged as paid hourly, so there is no qualifying overtime. It also left out the $2,000 non-itemizer charitable deduction and added the child's separate $3,220 tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly arrived at $5,200.24 for the joint return, then added the child's separate $3,220 tax. That is wrong because this output covers only the parents' tax unit."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint income and deducted mortgage interest even though no interest paid was listed. The correct calculation uses AGI of $100,002 and the $32,200 standard deduction plus the $2,000 charitable deduction, then subtracts the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied post-TCJA-sunset law for 2026 (restored exemptions, 15% bracket, and a $1,000 CTC) and deducted PA income tax as part of itemizing. It also added the child's separate $4,851 tax. Under OBBBA, the standard deduction plus the $2,000 charitable deduction applies, the brackets are 10%/12%, and the CTC is $2,200, on the parents' return only."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired: it used exemptions, 10/15/25% brackets and an unindexed $110,000 CTC phaseout that wiped out the credit. It also put the child's wages in the joint AGI. Under 2026 law the CTC is $2,200 with a $400,000 phaseout, and the parents' AGI is $100,002."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 law ($5,400 exemptions, 15%/25% brackets, and a CTC phaseout at $110,000) and included the child's $45,000 of wages in the joint AGI. Under OBBBA, the parents' $65,802 of taxable income is taxed at 10%/12%, and the full $2,200 CTC is subtracted."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA rules (three $5,434 exemptions, itemizing, the 15% bracket, a 10% medical floor, and a $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added the child's $4,846 separate tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It reached roughly the right $65,802 of taxable income and about $5,210 of tax for the parents, then added the child's separate return tax (about $3,225). That child tax is not part of the joint tax unit's federal income tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $12,000 qualified overtime deduction for Head, who is not flagged as paid hourly. It also included the child's $45,000 of wages in the joint AGI and left out the $2,000 non-itemizer charitable deduction, so it taxed $100,802 instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed credits wipe out the liability, but the parents' $7,400.24 of tax on $65,802 of taxable income far exceeds the $2,200 nonrefundable CTC. It also wrongly brought refundable credits into an output that is measured before refundable credits."
+us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into the joint AGI and took a $12,500 overtime deduction for Head, who is not flagged as paid hourly. It also left out the $2,000 non-itemizer charitable deduction, so taxable income came out at $100,302 instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's wages in a $145,002 AGI, made up $27,300 of mortgage interest from the balance, and used 2025 brackets with a $2,000 CTC. The correct calculation uses $100,002 of AGI, the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 brackets, and a $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up deductions and credits that do not apply: a QBI deduction on W-2 wages, mortgage interest that was not listed, a CDCC with no care expenses, and the foreign earned income exclusion. Its own math still left $5,687 of tax, yet it submitted 0. The correct result is $7,400.24 less the $2,200 CTC."
us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model paid out a flat $1,600 ""refundable CTC"" without running the ACTC limitation: the Child Tax Credit is first applied as a nonrefundable credit, and with MFJ taxable income of roughly $112,800 (AGI $145,002 less the 2026 MFJ standard deduction) generating about $14,000 of federal income tax, the entire CTC is absorbed nonrefundably and no additional child tax credit arises. It compounded this by inventing $205 of EITC at $145,002 of AGI, when the one-child EITC for joint filers is fully phased out in the upper $50,000s."
us,scenario_123,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model asserted that satisfying the 15%-of-earnings phase-in makes ""the full credit refundable,"" skipping the step that governs here: the CTC offsets the household's roughly $14,000 federal income tax liability as a nonrefundable credit first, and only the unused remainder — zero — flows to the refundable additional child tax credit, which is separately capped at $1,700 per child in 2026 rather than the full $2,400 the model reported."
us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model claimed ""no limitation from the nonrefundable portion,"" which is exactly backwards: the nonrefundable CTC is applied first against the household's roughly $14,000 of federal income tax before refundable credits, exhausting the credit and leaving $0 of additional child tax credit. Its $1,500 figure is also a stale pre-2022 ACTC cap rather than the 2026 $1,700 ceiling."
@@ -9879,94 +10118,97 @@ us,scenario_123,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_ou
us,scenario_123,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the household's MAGI was below Pennsylvania's adult Medicaid threshold, even though the trace places MAGI at 5.31 times FPL. It also treated being under age 65 as sufficient categorical status, while the head qualifies through no Medicaid pathway."
us,scenario_123,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model inferred Philadelphia residence solely from the household's Pennsylvania state and applied Philadelphia's wage tax despite the instruction to treat unlisted locality facts as false. Its unexplained reduction from the computed $5,613.24 to $3,545 also applies no identified exemption or valid computation step."
-us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Philadelphia as the default Pennsylvania locality and taxed both workers' wages even though Philadelphia residence or employment was never listed. It also submitted $3,830.90 after its own calculations produced $5,437.50 or $5,495.50, so the final number does not follow from its stated rate or tax base."
-us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right, including PA employee UC at 0.07% ($70 + $31.50), and summed them to exactly $11,194. It then submitted an unrelated $11,663.35, so the error is purely in transcribing the final answer."
-us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It applied 7.65% FICA to both earners ($11,092.50) but left out Pennsylvania's mandatory employee unemployment compensation contribution. That contribution is 0.07% of all $145,000 in wages and adds $101.50."
-us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It said outright that the head's $100,000 is below the Additional Medicare Tax threshold, then added a fabricated $648.89 of that tax anyway. It also claimed PA has no mandatory employee payroll tax, which drops the 0.07% UC contribution ($101.50) on $145,000 of wages."
-us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly worked out PA's employee UC contribution as 0.07% × $145,000 = $101.50, then put $0 into the sum and called it 'minimal.' The result was $11,092.50 instead of $11,194."
-us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning included Social Security, Medicare and PA employee UC at 0.07% and summed to exactly $11,194 twice. It then submitted $11,097.50, which matches none of its own figures, so this is a final-answer transcription error."
-us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It counted only federal FICA at 7.65% on the head's and child's wages ($11,092.50). It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all $145,000 of wages ($101.50)."
-us,scenario_123,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied 7.65% to the $145,000 of combined wages and stopped there. It missed Pennsylvania's mandatory 0.07% employee UC contribution, which adds $101.50."
-us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It recognized PA's 0.07% employee UC contribution but wrongly limited it to the first $10,000 of each worker's wages ($14 in total). Pennsylvania's employee UC withholding applies to all gross wages: $70 on the head and $31.50 on the child, $101.50 in all."
-us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It stated that PA has 'no state disability/UI employee withholding.' Pennsylvania in fact withholds 0.07% of all wages for employee unemployment compensation, which is $101.50 on $145,000, so the model stopped at $11,092.50."
-us,scenario_123,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It computed federal Social Security and Medicare correctly but declared that no employee state payroll tax applies. That leaves out Pennsylvania's mandatory 0.07% employee UC contribution on $145,000 of wages ($101.50)."
-us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only employee FICA on $145,000 ($8,990 + $2,102.50). It never applied Pennsylvania's 0.07% employee unemployment compensation contribution, which adds $101.50."
-us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It totaled Social Security and Medicare for the head and child ($11,092.50) and never considered state payroll tax. That misses Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It stated that no mandatory state payroll tax applies. Pennsylvania levies a 0.07% employee unemployment compensation contribution on all wages, which is $101.50 on $145,000."
-us,scenario_123,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only federal Social Security and Medicare on $145,000 ($11,092.50). It left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $145,000, which is $11,092.50, and then misreported the total as $11,108. It also left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It included only Social Security and Medicare, which are $11,092.50 on $145,000, then submitted $11,138, a figure its stated components do not produce. It never applied Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It summed federal FICA for the head ($7,650) and child ($3,442.50) and stopped. It missed Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on both workers' wages ($101.50)."
-us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It stated that 'PA has no mandatory employee-side state payroll tax.' Pennsylvania withholds 0.07% of all employee wages for unemployment compensation, which is $101.50 here."
-us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It claimed PA has no mandatory employee payroll tax, which leaves out the 0.07% employee UC contribution on $145,000 of wages ($101.50). Its total stopped at $11,092.50."
-us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It described only Social Security and Medicare on $145,000, which are $8,990 + $2,102.50 = $11,092.50, yet submitted $12,366, an overstatement of $1,273.50 that its own components do not produce. It also assumed no PA employee payroll tax, which drops the 0.07% UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It named only 6.2% Social Security and 1.45% Medicare on $145,000, which yield $11,092.50, but reported $15,399, an effective rate of 10.6% that no stated rule supports. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It applied 7.65% FICA to each earner and ignored state payroll tax. It missed Pennsylvania's mandatory 0.07% employee unemployment compensation contribution ($101.50).
-us,scenario_123,payroll_tax,gpt-6-astra,llm_error,thresholds_rates,False,"It correctly included the PA employee unemployment contribution on all $145,000 of wages but used a 0.06% rate ($87). The employee rate is 0.07%, which gives $101.50."
-us,scenario_123,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It asserted that no Pennsylvania employee payroll tax applies. PA withholds 0.07% of all wages for employee unemployment compensation, which adds $101.50 to the $11,092.50 of FICA."
-us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $145,000, which is $11,092.50, and misreported the sum as $11,107.50. It also left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It stated there is 'no mandatory PA employee state payroll tax.' That leaves out Pennsylvania's 0.07% employee unemployment compensation withholding on $145,000 of wages ($101.50)."
-us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,It claimed PA has no mandatory employee-side state payroll tax and stopped at 7.65% FICA. It missed the 0.07% employee UC contribution on all wages ($101.50).
-us,scenario_123,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It declared that Pennsylvania has no mandatory employee state payroll tax, so it left out the 0.07% employee unemployment compensation contribution on the head's and child's wages ($70 + $31.50)."
-us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It ruled out any mandatory state employee payroll tax. Pennsylvania's 0.07% employee UC contribution applies to all $145,000 of wages and adds $101.50."
-us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,"It set state mandatory employee payroll taxes to zero. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all wages, which is $101.50 here."
-us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there is no substantive answer to evaluate. The correct figure is $8,990 Social Security + $2,102.50 Medicare + $101.50 PA employee UC = $11,194."
-us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that the head's $100,000 and the child's $45,000 wages generate employee Social Security and Medicare tax, then reported $0 anyway, claiming the benchmark 'reports 0.' That drops all $11,194 of employee payroll tax, including the $101.50 PA UC contribution."
-us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that 'PA levies no mandatory employee payroll tax.' Pennsylvania's 0.07% employee unemployment compensation contribution on $145,000 of wages adds $101.50."
-us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It asserted that PA has no state payroll taxes. That leaves out the mandatory 0.07% employee UC contribution on the head's $100,000 and the child's $45,000 wages ($101.50)."
-us,scenario_123,payroll_tax,qwen3.8-max,llm_error,other,False,"It labeled the head's full 7.65% FICA ($7,650) as Social Security, then added Medicare ($1,450) again, which double-counts it. It invented a $270 Additional Medicare Tax even though the couple's $100,000 of wages is far below the $250,000 threshold. It also left out PA's 0.07% UC contribution and replaced the resulting sum with an unexplained $10,851.95."
+us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model assumed the household lives in Philadelphia and applied the 3.8712% resident wage tax to the Head's and Child 1's wages, but the facts list only PA with no locality. A PA household with no locality is outside every jurisdiction this output covers, so the liability is $0. Its final $3,545 also does not match its own figures ($3,871.20 + $1,742.04): it lowered the total by an arbitrary amount for a 'limited exemption' it never identified."
+us,scenario_123,local_income_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"The model applied a generic Pennsylvania municipal earned income tax of about 3.67% to $145,000 of combined wages. The output is limited to NYC, Philadelphia, Kansas City and St. Louis taxes, and no locality is listed for this household. Ordinary PA municipal and school-district EITs are not part of local_income_tax, so the correct value is $0."
+us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model said Philadelphia is 'the relevant PA locality' just because the household is in PA, and applied a resident wage tax of about 3.75% to $145,000 of wages. The household has no Philadelphia residence, so no covered local tax applies and the correct value is $0. Its submitted $3,830.90 also contradicts its own stated total of $5,437.50."
+us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got the right total of $11,194: FICA on both earners plus the 0.07% PA employee UC contribution ($70 + $31.50). It then submitted an unrelated $11,663.35 that matches nothing in its own work."
+us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA at 7.65% ($11,092.50). It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all $145,000 of wages, which adds $101.50."
+us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It said the head's $100,000 is below the Additional Medicare Tax threshold, then added $648.89 of that tax anyway. It also wrongly said Pennsylvania has no mandatory employee payroll tax, so it missed the $101.50 UC contribution at 0.07% of wages."
+us,scenario_123,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It named the right components (6.2% Social Security and 1.45% Medicare on $145,000), but those come to $11,092.50, not its unsupported $13,500. It also omitted Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly computed the PA employee UC contribution as $101.50 (0.07% × $145,000). It then wrongly treated it as $0 in the total and submitted $11,092.50 instead of $11,194."
+us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"It correctly derived $11,194 ($7,720 for the head plus $3,474 for the child, including PA UC at 0.07%), then submitted $11,097.50, which contradicts its own arithmetic."
+us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It counted only federal FICA at 7.65% ($11,092.50). It omitted Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all wages ($101.50)."
+us,scenario_123,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $145,000. It left out Pennsylvania's mandatory 0.07% employee UC contribution on all wages, which adds $101.50 to reach $11,194."
+us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"It recognized the PA employee UC contribution at 0.07% but capped it at the first $10,000 of wages per worker ($14). Pennsylvania's employee contribution has no wage cap, so it applies to the full $145,000 and comes to $101.50."
+us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly stated that Pennsylvania has no employee UI withholding. Pennsylvania requires a 0.07% employee unemployment compensation contribution on all wages, so its $11,092.50 is missing $101.50."
+us,scenario_123,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It said no employee state payroll tax applies, which omits Pennsylvania's mandatory 0.07% employee UC contribution on $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed only Social Security and Medicare ($11,092.50). It omitted Pennsylvania's 0.07% employee unemployment compensation contribution on all wages ($101.50)."
+us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,It summed only federal FICA for the head and child. It omitted Pennsylvania's mandatory 0.07% employee UC contribution ($70 + $31.50 = $101.50).
+us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It explicitly said no mandatory state payroll tax applies. Pennsylvania levies a 0.07% employee unemployment compensation contribution on all wages, which adds $101.50."
+us,scenario_123,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at Social Security plus Medicare ($11,092.50). It never added Pennsylvania's 0.07% employee UC contribution on $145,000 ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its stated 7.65% of $145,000 comes to $11,092.50, not $11,108, so its arithmetic is off. It also omitted Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare, which come to $11,092.50, and its $11,138 does not match that. It also omitted Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It treated household payroll tax as federal FICA only ($11,092.50). It omitted Pennsylvania's 0.07% employee unemployment compensation contribution on all wages ($101.50)."
+us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It wrongly stated that Pennsylvania has no mandatory employee-side payroll tax. It missed the 0.07% employee UC contribution on $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It asserted that Pennsylvania has no mandatory employee payroll tax and so dropped the 0.07% employee UC contribution ($101.50). Its result is federal FICA alone, $11,092.50."
+us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its stated components (6.2% OASDI and 1.45% Medicare on $145,000) come to $11,092.50, not its $12,366. It also explicitly assumed no PA employee payroll tax, which drops the 0.07% UC contribution ($101.50)."
+us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its $15,399 is far above what its own method produces: 7.65% of $145,000 is $11,092.50. It also excluded Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It computed only 7.65% FICA on the head's and child's wages. It omitted Pennsylvania's 0.07% employee unemployment compensation contribution on all wages ($101.50).
+us,scenario_123,payroll_tax,gpt-6-astra,llm_error,thresholds_rates,False,"It correctly included the PA employee UC contribution but computed it as $87, which is a 0.06% rate. Pennsylvania's employee rate is 0.07%, giving $101.50 on $145,000."
+us,scenario_123,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It explicitly said no Pennsylvania employee payroll tax applies. It missed the mandatory 0.07% employee UC contribution on $145,000 ($101.50)."
+us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its stated 7.65% on $145,000 is $11,092.50, not $11,107.50, so its arithmetic is wrong. It also omitted Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,It wrongly asserted that Pennsylvania has no mandatory employee payroll tax. That drops the 0.07% employee unemployment compensation contribution on all wages ($101.50).
+us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee-side payroll tax. Pennsylvania withholds a 0.07% employee UC contribution on all wages, which adds $101.50."
+us,scenario_123,payroll_tax,grok-4.7,llm_error,state_local_rule,False,It claimed Pennsylvania has no mandatory employee state payroll tax and so omitted the 0.07% employee UC contribution ($70 head + $31.50 child).
+us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It ruled out any mandatory state employee payroll tax. That misses Pennsylvania's 0.07% employee unemployment compensation contribution on $145,000 ($101.50)."
+us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,"It set state mandatory employee payroll taxes to zero. Pennsylvania imposes a 0.07% employee UC contribution on all wages, which adds $101.50."
+us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for payroll_tax, so nothing was available to score against the $11,194 total."
+us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that $145,000 of wages triggers employee Social Security and Medicare tax, then reported $0 anyway. That drops $8,990 of Social Security, $2,102.50 of Medicare, and the $101.50 PA UC contribution."
+us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,It wrongly asserted that Pennsylvania levies no mandatory employee payroll tax. That omits the 0.07% employee UC contribution on all wages ($101.50).
+us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It stated that Pennsylvania has no state payroll taxes, which misses the mandatory 0.07% employee unemployment compensation contribution on $145,000 ($101.50)."
+us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It recorded $7,650 (the head's full 7.65% FICA) as Social Security and then added $1,450 Medicare again, double-counting Medicare. It invented $270 of Additional Medicare Tax even though wages are well below the $250,000 MFJ threshold. Its final $10,851.95 does not match its own $10,812.50 sum, and it omitted the 0.07% PA employee UC contribution."
us,scenario_123,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible output, violating the required submission contract and omitting the engine-derived result of 0."
us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category."
us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It correctly computed the parents' joint PA tax as $3,070.06, then added $1,381.50 for a separate return for the dependent child, which falls outside this tax unit. The number it submitted ($4,453.06) also does not match its own sum of $4,451.56."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which counts the dependent child's $45,000 of wages. The tax unit's PA base is only the spouses' $100,002, so the correct tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented a $15,000 PA standard deduction and an $8,500 dependent deduction. Pennsylvania has neither: it taxes compensation and interest with no standard deduction. It also added the child's $45,000 of wages to the base, when the correct base is the spouses' $100,002 × 3.07%."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the parents' taxable compensation, making the base $145,002 instead of $100,002. It also submitted $4,452.08, which does not match the $4,451.56 it computed."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It included Child 1's $45,000 of wages in the PA base ($145,002) even though the joint return covers only the spouses' $100,002. It also submitted $4,452.61, which does not match its own computed $4,451.56."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It computed the head and spouse's tax as about $3,070, then added $1,381.50 for the child's own return. That separate return is outside this tax unit's liability, which is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the dependent child's wages with the parents' income. The joint return's base is the spouses' $100,002, giving $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It correctly found that the PA joint return covers only the spouses' $100,002 ($3,070.06) and that the child files separately. It then overrode this by reading 'one tax unit' as pooling everyone's income. A dependent's wages are never part of the parents' PA joint return."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly derived 100,002 × 0.0307 = $3,070.06 and correctly excluded the child's wages. It then submitted $3,060, citing 'compensation-only interest treatment nuances' that do not exist in PA law."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It computed the parents' unit correctly at $3,070.06, then added the child's separate $1,381.50. The requested output is the single tax unit's liability, not the sum of two filing units."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It taxed $145,002 at 3.07%, which includes the dependent child's $45,000 of wages. The tax unit's PA base is only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It added a separate $1,381.50 tax on the dependent child's wages to the head's tax. It also dropped the spouses' $2 of taxable interest, which PA taxes as a separate income class. The correct figure is $100,002 × 3.07% = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the child's $45,000 of wages into the parents' base. The joint return's base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,100 charitable donation from the head's income, but PA allows no charitable deduction against compensation. It also added the child's $45,000 of wages. The correct base is the full $100,002 of the spouses' income."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It summed the income of all three members ($145,002), including the dependent child's $45,000 of wages. PA's joint return taxes only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a 3% rate instead of PA's 3.07% flat rate. It applied that rate to $145,000, which includes the child's wages and omits interest. The correct figure is 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to 'total household taxable income' of $145,002, which includes the dependent child's wages. The tax unit's base is the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It correctly computed the parents' joint tax as about $3,070, then added $1,382 for the child's separate return. That return is not part of this tax unit's liability."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,385 equals 3.07% of about $110,260, which matches no PA base in this household. The spouses' compensation plus interest is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of 'total household' income, pooling the dependent child's wages. The correct base is only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes Child 1's $45,000 of wages. The PA joint return for this tax unit includes only the head and spouse's $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of compensation, which includes the dependent child's wages. The joint return's compensation is only the head's $100,000, which gives $3,070.06 once the $2 of interest is added."
-us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It correctly computed the joint return at $3,070.06, then added the child's separately filed $1,381.50. The output covers only the one tax unit's return."
-us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It pooled the child's $45,000 of wages with the parents' income to get a $145,002 base. The tax unit's PA base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It pooled all household wages, including the dependent child's, into the PA base. Its $4,807 implies a base of about $156,580, which is more than all listed income combined. The correct figure is 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It applied federal-style itemized deductions, which PA does not allow, and an effective rate of about 6.2% on all household wages. PA taxes the spouses' $100,002 at a flat 3.07%, giving $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the PA base ($145,002). The joint return taxes only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's wages. Its submitted $4,455.06 also does not equal 3.07% of that base ($4,451.56). The correct figure is 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling Child 1's wages into the parents' base. The tax unit's base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It taxed all $145,002 of household wages and interest. The dependent child's $45,000 is not on the parents' PA joint return, so the base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It taxed $145,002 of 'household' income, which includes the dependent child's wages. The correct base is only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,household_unit_or_filing_status,False,"It correctly taxed the parents' $100,002, then added the child's separate $45,000 return. The requested output is the single tax unit's liability of $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of wages, which includes the dependent child's $45,000. The joint return's base is the head's $100,000 plus $2 of interest."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of wages, which includes the dependent child's $45,000. The tax unit's PA base is only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It taxed about $145,000 of combined household income, pooling the dependent child's wages. The correct base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of combined compensation, which includes the dependent child's wages. The joint return's base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It correctly ruled out PA deductions, but it put the child's $45,000 of wages in the tax unit's base ($145,002) instead of using only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's wages. The parents' joint return, the only return in this tax unit, has a base of $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It explicitly added the child's $45,000 of wages to the head and spouse's income. The dependent's wages are not on the parents' PA joint return, so the base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It correctly identified two filing units, then summed tax across both. The output is the tax unit's liability, which is the parents' return alone: $100,002 × 3.07% = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. $2,939.86 is 3.07% of about $95,761, so it took roughly $4,241 off the spouses' $100,002. PA allows no deduction against compensation for any of the listed expenses."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the dependent child's $45,000 of wages. The joint return's base is $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It stated that PA tax liability is positive, then submitted 0, contradicting its own reasoning. It also cited a PA standard deduction that does not exist. The correct figure is the full 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It correctly disallowed the charitable, medical and property-tax deductions, but it included the child's $45,000 of wages. The tax unit's base is only the spouses' $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It added Child 1's $45,000 of wages to the head's $100,000 of compensation. The PA joint return covers only the spouses' $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($67,525), but PA taxes gross compensation and interest with none of the federal standard or itemized deductions. It then submitted $2,565.30, which matches neither its own $2,073.02 nor the correct 3.07% × $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed the parents' joint PA tax of $3,070.06. It then added $1,381.50 for a separate child return, even though the prompt puts the child in the parents' single tax unit as a dependent whose wages are not taxed on that return. It also submitted $4,453.06, which does not match its own sum of $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by pooling the dependent child's $45,000 of wages into the tax unit's PA base. Only the head's and spouse's $100,002 is PA taxable income for this unit, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model invented PA standard deductions of $15,000 and $8,500, but PA allows no standard deduction or personal exemption. It also included the dependent child's $45,000 of wages. The correct base is the parents' $100,002 at 3.07%, which is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a base of roughly $145,000 that includes the dependent child's wages. Its $5,900 also does not match 3.07% of its own base, which is $4,451.56. The tax unit's PA base is only the parents' $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the parents' compensation, giving a $145,002 base. The dependent's wages are not part of the joint tax unit's PA income. It also submitted $4,452.08 instead of its own computed $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model included Child 1's $45,000 of wages in a combined $145,002 base. The tax unit's PA taxable income is only the head's and spouse's $100,002, so the tax is $3,070.06. It also submitted $4,452.61, which differs from its own computed $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly found about $3,070 for the head and spouse. It then added $1,381.50 of separate tax on the child's wages, even though the child is a dependent in the same single tax unit and no separate child return exists."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by counting the dependent child's $45,000 of wages as tax-unit income. The PA base is only the parents' $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly worked out that the joint return covers $100,002 and owes $3,070.06. It then overrode that result and added the child's $45,000 because 'all are listed in one tax unit.' Being in one tax unit is exactly why the dependent's wages are excluded from the parents' return."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly limited the base to the parents' $100,002 and computed $3,070.06. It then arbitrarily reduced this to $3,060, citing nonexistent 'compensation-only interest treatment nuances.' PA taxes interest at the same flat 3.07%."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the parents. It then added a 'separate unit' tax of $1,381.50 for the child, but the prompt places the child in the same single tax unit as a dependent, so no child return is taxed."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 by including the dependent child's $45,000 of wages. The tax unit's PA income is the parents' $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added $1,381.50 of tax on the dependent child's wages, which are not part of the parents' tax unit. It also dropped the couple's $2 of taxable interest. The correct figure is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by pooling the dependent child's wages into the tax unit's base. Only the head's and spouse's $100,002 is taxed, which gives $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $16,100 of charitable donations, but PA allows no charitable or itemized deductions. It also added the dependent child's $45,000 of wages. The correct base is the parents' $100,002 with no deductions, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed $100,001, $1, and the child's $45,000 into a $145,002 base. The dependent child's wages are not income of the parents' tax unit, so the base is $100,002 and the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a 3% rate instead of PA's 3.07% flat rate. It applied that rate to $145,000, which includes the dependent child's wages. The correct calculation is 3.07% × the parents' $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated $145,002 of 'household' wages and interest as the PA base, which pulls in the dependent child's $45,000. The tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed about $3,070 on the parents' joint $100,002. It then added $1,382 for a child return that does not exist, because the child is a dependent inside the single tax unit."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation, and its $3,385 implies a base of about $110,260 at 3.07%, which matches no income figure in the household. The correct calculation taxes only the parents' $100,002 of compensation and interest at 3.07%, which is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to a $145,002 'household' total that includes the dependent child's wages. Only the parents' $100,002 is the tax unit's PA income, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model used $145,002 as taxable income by counting the dependent child's $45,000 of wages. The joint tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted $145,000 of compensation, which includes the dependent child's wages that are not on the parents' return. The PA base is $100,000 of wages plus $2 of interest, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the joint return. It then added $1,381.50 for a separate child filing, but the prompt places the child in the single tax unit as a dependent, so no separate child tax is computed."
+us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 by adding the dependent child's $45,000 of wages to the parents' income. The tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed household-wide wages, including the dependent child's, and its $4,807 matches no correct product of 3.07% and any listed income total. The correct figure is 3.07% × the parents' $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model ran a federal-style itemized-deduction computation, but PA has no itemized deductions and taxes income at a flat 3.07%. Its $9,055 is nearly triple the correct $3,070.06, which is the tax on the parents' $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's $45,000 of wages in a $145,002 base. The joint tax unit's PA income is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's wages in a $145,002 base. It also misstated the product as $4,455.06 when the correct product is $4,451.56. The correct tax is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002, which includes the child's $45,000 of wages. A dependent's wages are not income of the parents' tax unit, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 of 'household' wages and interest, pulling in the dependent child's earnings. The tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 of household wages and interest, but the dependent child's $45,000 belongs outside the joint tax unit's PA income. The correct calculation is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly taxed the parents' $100,002. It then added a separate tax on the child's $45,000 of wages, even though the prompt puts the child in the single tax unit as a dependent with no separate return."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,000 of wages, which includes the dependent child's $45,000, which is not on the parents' return. The correct base is $100,000 of wages plus $2 of interest, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's $45,000 of wages in the PA base. Only the head's and spouse's $100,002 is taxed for this tax unit, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed a combined base of about $145,000 that includes the dependent child's wages. The joint tax unit's PA income is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed combined compensation of $145,002 by counting the dependent child's wages. Only the parents' $100,002 is the tax unit's PA base, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly noted that PA has no deductions. It then set taxable income at $145,002 by including the dependent child's wages, which are not income of the parents' tax unit. The correct tax is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model used $145,002 as taxable income, which includes the child's $45,000 of wages. The dependent's earnings are excluded from the joint tax unit, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the head's and spouse's income. The PA base for the single tax unit is only the parents' $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed 'both filing units,' treating the child as a separate filer. The prompt defines one tax unit in which the child is a dependent and the child's wages are not taxed, so the answer is the parents' $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its $2,939.86 implies a base of about $95,760, roughly $4,242 below the parents' $100,002. That shortfall shows it subtracted a deduction or exclusion that PA does not allow. The correct tax is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly applied no deductions but used $145,002 as taxable income by including the dependent child's wages. The tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model acknowledged that PA liability is positive but submitted 0, deferring to an imagined structured output. It also wrongly cited a PA standard deduction. The correct calculation is the flat 3.07% on the parents' $100,002, which is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly allowed no deductions but taxed $145,002 by including the dependent child's $45,000 of wages. The joint tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the parents' compensation, giving a $145,002 base, and also misrounded the result to $4,451.57. The dependent's wages are excluded from the tax unit, so the tax is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model used federal taxable income of $67,525 as the PA base, but PA does not start from federal taxable income. PA taxes gross compensation and interest with no federal deductions. The model also submitted $2,565.30, which contradicts its own $2,073.02. The correct base is $100,002, giving $3,070.06."
us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
index b82ee14e..3b12cc9d 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
@@ -1,5 +1,5 @@
country,scenario_id,variable,wrong_model_count,case_failure_sources,case_failure_subtypes,reference_suspect,reference_bug_hypothesis,case_annotation
-us,scenario_000,federal_income_tax_before_refundable_credits,42,llm_error,household_unit_or_filing_status,False,,"A 77-year-old surviving spouse with no dependent child cannot use Qualifying Surviving Spouse status, so the return is filed as Single. Taxable Social Security uses the $25,000/$34,000 thresholds: $4,500 + 0.85 × ($50,520 − $34,000) ≈ $18,542. That puts AGI at about $50,437. Under OBBBA-extended law, deductions are the $16,100 single standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, totaling $24,150. That leaves $26,287 of taxable income, taxed at 10% up to $12,400 and 12% above, for $2,906.45. Most wrong answers made one of three mistakes: they used joint (QSS) thresholds, deductions and brackets; they assumed the TCJA sunset brought back personal exemptions and the 15% bracket; or they left out the $6,000 senior deduction."
+us,scenario_000,federal_income_tax_before_refundable_credits,43,llm_error,household_unit_or_filing_status,False,,"The 'is a surviving spouse' flag does not create qualifying-surviving-spouse status. IRC §2(a) requires the filer to maintain a home for a dependent child, and none is listed, so the head files as single. Under single status, the $25,000/$34,000 thresholds make $18,542.70 of Social Security taxable, so AGI is $50,437.11. The post-OBBBA single standard deduction is $16,100 plus the $2,050 aged add-on, for $18,150, and the $6,000 senior deduction also applies. That leaves $26,287.11 of taxable income, taxed under the 2026 single brackets: 10% up to $12,400 and 12% above it, for $2,906.45. Most wrong answers made one of three errors: they used joint-style QSS thresholds and deductions, assumed the TCJA sunset (personal exemption and a 15% bracket), or left out the $6,000 senior deduction."
us,scenario_000,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_000,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Texas Medicaid eligibility at age 77 requires qualification through an aged Medicaid pathway; age alone does not establish a qualifying category. This person has no Medicaid eligibility category, receives no SSI, and has MAGI income of 3.98 times FPL, so the immigration-status requirement does not create eligibility by itself."
us,scenario_000,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wage and salary compensation, not to Social Security retirement benefits, pensions, IRA distributions, interest, or veterans benefits. Because the household reports no wages or other employee compensation, the employee Social Security and Medicare tax base is zero and payroll tax is $0."
@@ -13,30 +13,30 @@ us,scenario_001,state_income_tax_before_refundable_credits,7,llm_error,state_loc
us,scenario_002,federal_income_tax_before_refundable_credits,10,prompt_ambiguity,taxable_income_or_deductions,False,,"The correct 2026 AGI is $23,130: pension $18,340 + interest $400 + $4,390 of taxable Social Security. The $24,000 disability benefits and $48,000 veterans benefits are excluded from gross income, so provisional income is $18,740 + half of $44,080 = $40,780, which sits between the MFJ $32,000 and $44,000 thresholds and therefore includes only 50% × ($40,780 − $32,000) = $4,390 of benefits. That AGI is below the 2026 MFJ standard deduction alone ($32,200 base + two $1,650 age-65 additions = $35,500), and further below it once the $6,000-per-senior deduction ($12,000) is applied, so taxable income and tax before refundable credits are $0, and NIIT is $0 because AGI is far under $250,000. Nine models manufactured taxable income by pulling the non-taxable disability benefits into gross/provisional income (which also flips Social Security from the 50% tier to the 85% tier), by taxing 100% of Social Security, or by filing the couple as a single individual. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
us,scenario_002,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits, pensions, disability benefits, veterans benefits, or taxable interest. Because the household has no wages or other payroll-taxable earnings, every employee-side payroll-tax component is zero."
us,scenario_002,spouse_medicaid_eligible,1,llm_error,household_unit_or_filing_status,False,,"Medicaid eligibility for the spouse cannot be established by treating the spouse's lack of individually reported income as zero countable income. The spouse qualifies through no Washington Medicaid pathway: the engine assigns category NONE, records no SSI, and calculates household MAGI at 2.90 times FPL, above the applicable MAGI limits."
-us,scenario_003,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine subtracts the $1,118 traditional IRA contribution above the line without applying the IRC §219(b)(1)(B) limit, which caps the deduction at compensation includible in gross income, or the §219(c) spousal limit. Both spouses have $0 wages and $0 self-employment income, so the correct deduction is $0; allowing it understates AGI by $1,118 and tax by about $246.","The reference builds AGI of $182,172.27 by subtracting both the $1,582 net capital loss and the $1,118 traditional IRA contribution from gross income. That gross income includes 85% of Social Security, the IRA distributions, the pension, the interest and the dividends. It then subtracts the 2026 MFJ standard deduction of $32,200, taxes the $146,212.27 ordinary portion at 10/12/22% with thresholds of $24,800 and $100,800 ($21,590.70), and adds 15% on the $3,760 of qualified dividends. The strongest cluster of models matched every step except the IRA deduction, which they refused because there is no compensation, and landed at $22,400.71. The remaining models made their own errors: TCJA-sunset personal exemptions and 15%/25% brackets, 2025 or guessed brackets and standard deductions, a 401(k) deduction with no wages, dropped or mis-taxed Social Security, double-counted capital items, or submitted values that did not match their own work. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 22,154.70; the corrected value is 22,400.65. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)."
-us,scenario_003,head_medicare_eligible,3,llm_error,age_disability,False,,"Standard Medicare eligibility requires the person to be at least age 65 unless a separate disability or disease-based pathway applies. The head is 64 throughout 2026, and all unlisted disability and qualifying disease statuses are false, so neither current nor projected eligibility applies."
+us,scenario_003,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"To match the reference, a model has to: include 85% of Social Security; net the $1,080 short-term gain against the $2,662 long-term loss for a $1,582 deduction; take no traditional 401(k)/IRA deduction because nobody has compensation; and apply the 2026 rules made permanent by OBBBA. Those rules are a $32,200 joint standard deduction (no age-65 or senior add-on, since the head is 64 and the spouse 61), a 10% bracket to $24,800, a 12% bracket to $100,800 and 22% above that, with $3,760 of qualified dividends taxed at 15%. The largest group of wrong answers assumed the TCJA expired, bringing back personal exemptions and the 10/15/25% brackets. Another group used 2025 or earlier standard deductions and bracket limits. The rest invented deductions (retirement contributions without earnings, car-loan interest, age-65 amounts), double-counted capital gains or losses, or submitted a number that did not match their own working."
+us,scenario_003,head_medicare_eligible,4,llm_error,age_disability,False,,"Medicare eligibility without disability or ESRD requires age 65. The head is 64 for the whole 2026 tax year, and the prompt says ages and statuses stay constant all year. The head is listed with no disability, no SSDI, and no ESRD, so no non-age pathway applies. Two models correctly reasoned ""64 < 65, not eligible"" but still submitted 1. The other two invented a ""near-term"" or ""projected"" eligibility rule that doesn't exist."
us,scenario_003,payroll_tax,2,llm_error,payroll_tax_base,False,,"Usual weekly hours and employer-sponsored insurance establish work and coverage facts but do not supply wage income or an hourly rate. Because unlisted numeric inputs are zero, both people have zero wages subject to employee Social Security and Medicare taxes, so employee payroll tax is zero."
us,scenario_003,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no state individual income tax, and exclusive Texas residency makes every other state's income-tax component zero. The aggregate state income tax before refundable credits is therefore the sum of 51 zero components, yielding $0."
us,scenario_004,federal_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"The case turns on the IRC §86 provisional-income test: provisional income is $1,200 interest − $135 partnership loss + half of the $51,475 Social Security benefit = $26,802.50, which sits below the $32,000 MFJ base amount, so none of the Social Security enters gross income. AGI is therefore $1,065, far below the 2026 MFJ standard deduction, leaving $0 taxable income and $0 income tax before credits; the $1,200 of interest is nowhere near the $250,000 MFJ NIIT threshold, and the traditional 401(k)/IRA contributions are irrelevant because the household has no earned income to support them. Every wrong model produced a positive tax by pulling Social Security benefits into taxable income — fully, at 85%, or by keeping them in the AGI it then reduced by the standard deduction — except Sonnet, which reached $0 and then overrode it with imputed wages the facts do not list."
us,scenario_004,head_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"The head’s Medicaid MAGI is approximately $52,540, or 2.39 times FPL, because the income calculation includes the applicable Social Security retirement income rather than reducing MAGI to taxable interest less the partnership loss. At that income level the head qualifies through neither New York’s expansion-adult category nor any non-MAGI categorical pathway, so medicaid_category is NONE."
-us,scenario_004,head_medicare_eligible,4,llm_error,age_disability,False,,"Standard age-based Medicare eligibility begins at age 65, and the head is 62. Social Security retirement income does not independently establish Medicare eligibility, and no disability, end-stage renal disease, or ALS pathway is listed."
+us,scenario_004,head_medicare_eligible,5,llm_error,age_disability,False,,"Medicare eligibility starts at age 65. Before 65, a person qualifies only after 24 months of Social Security disability benefits or because of ESRD/ALS. The head is 62, has no listed disability, and does not qualify through any of those routes. Taking reduced Social Security retirement benefits early at 62 does not make someone eligible for Medicare. Every wrong model either treated Social Security retirement income as a path to Medicare or misapplied the age-65 test."
us,scenario_004,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other covered earned compensation, not Social Security retirement benefits, taxable interest, or partnership/S-corporation losses. Because unlisted wages and hourly rates are zero, the household has no employee payroll-tax base and employee payroll tax is $0."
us,scenario_004,spouse_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"New York's ACA expansion pathway requires the spouse's MAGI income level to be at or below 138% of FPL. The household's applicable MAGI is 2.39 times FPL, and the 51-year-old spouse has no disability, blindness, SSI, age-based, dependent, or other categorical pathway, so the Medicaid category is NONE."
us,scenario_004,state_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"Both spouses show 40-hour work weeks but no wage amount, and the prompt directs that unlisted numeric inputs be treated as 0, so the household's only non-Social-Security income is $1,200 of taxable interest less a $135 partnership loss = $1,065. Provisional income is $1,065 + half of $51,475 = $26,803, below the $32,000 married-filing-jointly base, so none of the Social Security benefit is federally taxable, and New York subtracts any taxable Social Security from federal AGI regardless. New York AGI of $1,065 is entirely absorbed by the ~$16,050 MFJ standard deduction, leaving $0 of NY taxable income and $0 of tax before refundable credits. Every wrong answer either taxed the $51,475 Social Security benefit, imputed phantom wages from the hours-worked fact, or both."
us,scenario_004,state_refundable_credits,1,llm_error,state_local_rule,False,,"This couple has no wages or rent, so its federal AGI is only about $1,065: $1,200 of interest minus a $135 partnership loss. None of the $51,475 in Social Security is taxable, because provisional income is below the $32,000 joint base. The only NY credit this low income could trigger is the household credit under Tax Law §606(b), and that credit is nonrefundable. It can only reduce NY tax, which is already zero, and it never counts toward state_refundable_credits. None of New York's refundable credits apply either: there is no federal EITC, no qualifying child, and no property tax or rent for the real property tax credit, so the refundable total is $0."
-us,scenario_005,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference uses 2026 law after OBBBA, which made the TCJA rules permanent. AGI is $536,765. Each spouse's traditional 401(k) deferral is scaled pro rata to $20,825 so that traditional plus Roth stays within the $24,500 combined limit. The frozen reference allows the $2,164 IRA deduction, excludes the $4,148 state refund and caps the net capital loss at $3,000. It allows the IRA deduction because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion: under 26 U.S.C. 219(g) both spouses' traditional 401(k) deferrals make them active participants, their modified AGI for 219(g), $543,076, is far above $149,000, the top of the 2026 joint phase-out range, and the exclusion's corrected value, $107,198.34, allows no IRA deduction. The deduction is the $32,200 MFJ standard deduction plus the new $1,070 cash-charity deduction for non-itemizers, giving taxable income of $503,495. Tax is $89,096 at 2026 brackets, plus 15% on $77,920 of qualified dividends and 3.8% NIIT on $150,578, which this variable includes; there is no AMT and no mortgage interest deduction. Wrong answers mostly (a) assumed TCJA sunset rules such as personal exemptions, Pease, miscellaneous deductions, pre-TCJA brackets and AMT, (b) imputed mortgage interest from the balance even though no interest was listed, (c) left out NIIT, or (d) came within a few hundred dollars but missed the $1,070 non-itemizer deduction, the 401(k) proration or the refund exclusion; several also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_005,federal_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"PolicyEngine applies the 2026 rules under OBBBA, which made the TCJA rules permanent. AGI is $536,764.50: wages less traditional 401(k) deferrals, where each spouse's $27,240 traditional+Roth request is cut to the $24,500 combined limit, pro-rating traditional deferrals to $20,825 each ($41,650 total); plus the $2,164 IRA deduction, the $3,000 capital-loss deduction, and no taxable state refund. The household takes the $32,200 standard deduction plus the $1,070 non-itemizer cash charitable deduction, because the phased-down SALT cap (~$30.9k) and the charity gift, which the 0.5%-of-AGI floor wipes out, cannot beat it. Tax is $89,095.92 of ordinary bracket tax, plus 15% on $77,920 of qualified dividends ($11,688), plus 3.8% NIIT on $150,578.34. The wrong answers did one or more of these: applied TCJA-sunset law (pre-TCJA brackets, Pease, personal exemptions, AMT), invented mortgage interest from a balance with no interest given, left out NIIT, subtracted ESI premiums from wages, or, among the near-misses, missed the 401(k) pro-rating, the refund exclusion, and the non-itemizer charitable deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_005,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_005,payroll_tax,42,llm_error,state_local_rule,False,,"The reference total is $36,044. It adds employee Social Security of $22,599 (6.2% × ($180,000 + the $184,500 2026 wage base)), Medicare of $6,235 (1.45% × $430,000), and Additional Medicare Tax of $1,620 (0.9% × ($430,000 − $250,000 MFJ threshold)), for $30,454. It then adds California SDI of $5,590, which is the 2026 rate of 1.3% on all $430,000 of wages with no wage ceiling. The main mistake was California SDI: many models left it out, used last year's 1.1% or 1.2% rate, or applied the wage cap that ended in 2024. Other errors were using a pre-2026 Social Security wage base (capping the head's $180,000) and subtracting employer health premiums from FICA/SDI wages, even though the facts give gross wages as the payroll tax base."
+us,scenario_005,payroll_tax,43,llm_error,state_local_rule,False,,"The federal parts were easy to get right: Social Security is 6.2% of $180,000 plus 6.2% of the $184,500 2026 cap on the spouse's $250,000, totaling $22,599. Medicare is 1.45% of $430,000, or $6,235. Additional Medicare Tax is 0.9% of the $180,000 above the $250,000 MFJ threshold, or $1,620. The trap that separated the answers is California SDI. In 2026 it is 1.3% of all gross wages with no wage ceiling, so it is $5,590 on $430,000. Most models left SDI out, used the old 1.1%/1.2% rates, applied the wage cap that was repealed in 2024, or subtracted employer-sponsored insurance (ESI) premiums from the payroll wage base, which the engine taxes in full."
us,scenario_005,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_005,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's 2026 CA itemized base is exactly the $17,280 of real estate taxes ($17,280 − 6% × ($536,764.50 − $504,411) = $15,338.79 to the cent), so it omits the $2,574 of charitable gifts — consistent with passing CA the federal charitable deduction after OBBBA's 0.5%-of-AGI floor, which California has not adopted under its January 1, 2015 IRC conformity date (R&TC §17024.5) — and omits unreimbursed employee expenses above 2% of AGI, which stay deductible on Schedule CA (540) because California never conformed to IRC §67(g); including both gives ~$21,025 of itemized deductions and ~$40,523 of tax.","California taxable income is $521,425.72. CA AGI is $536,764.50: federal AGI minus the $4,148 state refund. Each spouse's combined traditional+Roth 401(k) request of $27,240 is capped at the $24,500 limit, the $2,164 of IRA deductions are taken, and employer insurance premiums, tax-exempt interest, the tax-exempt pension and mortgage interest (none is listed) are all left out. From that AGI come $15,338.79 of itemized deductions: the $17,280 of real estate taxes cut by 6% of AGI above $504,411, which beats the $11,412 standard deduction. The frozen reference's projected 2026 MFJ schedule taxes this income at a top rate of 9.3%, giving $41,208.45, and its projected MFJ exemption credits, reduced by $6 per $2,500 of excess AGI, leave $156.93 to subtract. The release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $41,369.87 of tax and $150 of credits on the same income. The wrong answers come mostly from imputing mortgage interest on the $2.85M balance, subtracting employer health premiums or adding tax-exempt income, excluding the full 401(k) request, putting charity and employee expenses into the itemized base or skipping the limitation, zeroing or over-sizing the exemption credits, and misapplying the brackets. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 41,051.51; the corrected value is 40,920.40. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)."
+us,scenario_005,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The answer depends on getting California taxable income right. Each spouse wants $23,154 traditional plus $4,086 Roth 401(k), or $27,240 in total, which is over the $24,500 2026 elective-deferral limit. PolicyEngine pro-rates the cap, so only about $20,825 of traditional deferral per spouse comes out of wages. That puts federal AGI at $543,076, and California AGI is $538,928 after the $4,148 SALT-refund subtraction. California itemized deductions are property tax $17,280, charity $2,574 and employee business expenses above the 2% floor ($2,986). There is no mortgage interest because none is listed, and the tax-exempt pension and interest are not added back. The 6% high-income limitation cuts these deductions to $20,520, so taxable income is $518,408 and tax is $41,089. Only $114 of the exemption credit survives the phase-out. Most models deducted the full $46,308 of 401(k) deferrals, made up mortgage interest, subtracted employer-paid premiums, skipped or misapplied the itemized limitation, or treated the exemption credit as either fully phased out or not phased out at all."
us,scenario_005,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_007,federal_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"Getting the reference takes two steps. First, apply the IRC §86 two-tier formula to the SSDI. Provisional income is $35,280 + $9,642.50 = $44,922.50, and it excludes the $284 tax-exempt pension, which is not tax-exempt interest. Taxable SS is the lesser of 85% of benefits or 0.85×(PI − $34,000) + min($4,500, 50% of benefits), which gives $13,784.12 and AGI of $49,064.12. Second, use the 2026 parameters that OBBBA made permanent: a $16,100 single standard deduction and a 10% bracket through $12,400. That gives taxable income of $32,964.12 and tax of $1,240 + 12% × $20,564.12 = $3,707.70, with no nonrefundable credit available. The wrong answers split into two groups. Some broke the §86 step: they taxed a flat 85% of the SSDI, used only the 50% tier, added the tax-exempt pension to provisional income, or left the SSDI out. The others used out-of-date deduction and bracket figures: 2024 or 2025 amounts, rough inflation guesses, or pre-TCJA law with a personal exemption and a 15% bracket. A few derived a figure and then submitted a different, unsupported number. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
+us,scenario_007,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The correct answer needs three things. First, taxable SSDI is $13,784.12: provisional income is $35,280 + $9,642.50 = $44,922.50, and the taxable amount is the lesser of 85% × $19,285 and 0.85 × ($44,922.50 − $34,000) + $4,500. The tax-exempt pension is not counted in provisional income. Second, the 2026 single standard deduction is $16,100, because OBBBA made the TCJA rules permanent, so there is no personal exemption. Third, the 2026 brackets are 10% up to $12,400 and 12% above that, which gives $1,240 + 12% × $20,564.12 = $3,707.70. The wrong answers fall into four groups: some applied the SS taxation formula incorrectly (taking the 85% cap instead of the lesser amount, using only the 50% tier, adding the $284, or treating SSDI as untaxed); some used stale 2024/2025 deductions and brackets; some assumed the TCJA had expired, with personal exemptions and a 15% bracket; and some submitted a value that differs from their own calculation. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
us,scenario_007,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_007,head_medicare_eligible,22,prompt_ambiguity,age_disability,False,,"The head is 56, so Medicare's age-65 test fails, and the only other route is disability entitlement. Under 42 U.S.C. 426(b), that route requires 24 months of SSDI entitlement, not just SSDI income in the current year. The household lists no disability status, no months of SSDI receipt, no ESRD and no ALS. The prompt says unlisted status inputs are false and unlisted numbers are 0, and a constant status across one tax year shows at most 12 months. All 22 models treated the $19,285 of SSDI income as proof of Medicare eligibility, either by assuming the 24-month wait was already met or by claiming PolicyEngine grants eligibility whenever SSDI is positive, so they answered Yes instead of No. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_007,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other FICA-covered earned compensation, not to Social Security disability benefits or private pension income. The prompt specifies no wage or self-employment income and requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and payroll tax is $0."
us,scenario_007,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_007,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"Getting to Idaho taxable income is simple. Federal AGI is $49,064.12. Idaho subtracts all $13,784.12 of federally taxable Social Security, leaving $35,280, and the federal-conforming 2026 standard deduction of $16,100 brings the frozen reference's taxable income to $19,180. The frozen reference leaves out Idaho's subtraction of the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes, for taxable income of $17,100. The step that separates the reference from most wrong answers is Idaho's 2026 zero-rate band. The frozen reference taxes the first $4,920 of single taxable income, the engine's projected band, at 0% and only the remaining $14,260 at 5.3%, giving $755.78. The exclusion's corrected value uses the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026, for 5.3% × ($17,100 − $4,811) = $651.32, which gpt-6-astra matches. Most models reached $19,180 but taxed it from the first dollar, at 5.3% ($1,016.54) or at the old 5.695%/5.8% rates. Others made earlier mistakes: they skipped the Social Security subtraction, used a pre-OBBBA or TCJA-sunset standard deduction, or invented exclusions. Near misses differ from the frozen reference only in the zero-rate band: a guessed one ($4,990 or $5,000), or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)."
-us,scenario_007,state_refundable_credits,36,llm_error,thresholds_rates,False,,"Idaho's only refundable state credit for this household is the Grocery Credit. For 2026 it is $155 for a resident under 65. It has no income phaseout, and it counts as refundable whether or not the filer owes Idaho tax. The head received no SNAP, so all 12 months qualify and the full $155 is refundable. Eighteen models knew the credit exists but used an outdated amount: $100 (pre-2023), $120 (2023) or $140 (the old 65+ amount, or an invented inflation adjustment). Seventeen models answered $0. They said the credit is nonrefundable, used up against tax liability, phased out by income, repealed, or that Idaho has no income tax. One model returned no answer."
+us,scenario_007,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,,"Idaho starts from federal AGI of $49,064.12 and subtracts the $13,784.12 of federally taxable Social Security, leaving $35,280. The $16,100 standard deduction brings Idaho taxable income to $19,180. For 2026 Idaho taxes income at 5.3%, but the first $4,920 for a single filer is taxed at zero, so the tax is 0.053 × ($19,180 − $4,920) = $755.78. Most models got to $19,180 and then left out the zero-rate amount, used the old 5.695% or 5.8% rate, or used an out-of-date zero-rate amount. Others never subtracted Social Security, used the wrong standard deduction, or wrongly applied Idaho's retirement-benefits deduction to a private pension for someone aged 56. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)."
+us,scenario_007,state_refundable_credits,37,llm_error,thresholds_rates,False,,"The whole $155 is the Idaho Grocery Credit. It is fully refundable and has no income phase-out. It is reduced only for months when the person gets SNAP or is incarcerated, and this filer qualifies for all 12 months. Idaho's 2025 legislation (HB 231) raised the per-person amount for filers under 65 from $120 to $155 starting in tax year 2025. The wrong answers split into two groups. One group used outdated per-person amounts: $120 under 2023–2024 law, $100 under earlier law, or $140, which was the old age-65+ amount. The other group answered $0, treating the credit as nonrefundable, phased out by income, repealed, or nonexistent."
us,scenario_008,child1_chip_eligible,9,llm_error,health_coverage,False,,"CHIP is a residual program: it covers children who do NOT qualify for Medicaid, so its income band has a floor at the state's children's Medicaid limit, not just a ~350% FPL ceiling. With household income of about $30,915 for a family of eight — roughly 54% of FPL — the 17-year-old clears New Jersey's Medicaid test in the OLDER_CHILD category, and that Medicaid eligibility forecloses CHIP. All nine models tested only the upper CHIP ceiling and read very low income as evidence for CHIP, when low income is exactly what routes the child to Medicaid instead."
us,scenario_008,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A 17-year-old tax-unit dependent falls within New Jersey's OLDER_CHILD Medicaid category. The household's MAGI is 0.55 times FPL, below the category's income threshold, so the child is eligible without any additional eligibility trigger."
us,scenario_008,child1_wic_eligible,2,llm_error,categorical_eligibility,False,,WIC child eligibility is restricted to infants and children younger than age five; a 17-year-old is categorically ineligible regardless of household income. Both models incorrectly treated age 17 as within the WIC child age range.
@@ -54,51 +54,52 @@ us,scenario_008,child5_chip_eligible,11,llm_error,health_coverage,False,,"CHIP i
us,scenario_008,child5_early_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Early Head Start serves infants and toddlers under age 3, while Head Start is the distinct program for preschool-age children. Child 5 is age 5 and therefore fails the Early Head Start age requirement regardless of household income or housing-assistance receipt."
us,scenario_008,child5_head_start_eligible,5,llm_error,age_disability,False,,"Every model rejected Child 5 on age alone, treating preschool Head Start as serving only 3- and 4-year-olds. Under 45 CFR 1302.12(b)(1) the range runs from age 3 up to the age of compulsory school attendance, which is 6 in New Jersey, and PolicyEngine's Head Start age window includes 5-year-olds. The age test is met. The child is also income-eligible, both categorically through the household's SNAP receipt and directly because household income of about $30.6K is well under the poverty-guideline threshold for a family of 8. So the answer is eligible. No model reached the income or categorical test, because each stopped at an age cutoff that was set too low."
us,scenario_008,child5_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A five-year-old in New Jersey falls within the YOUNG_CHILD Medicaid category. At MAGI equal to 0.55 times FPL, the child satisfies that category's income test and is Medicaid eligible."
-us,scenario_008,child5_wic_eligible,12,llm_error,age_disability,False,,"WIC's categorical definition of a child covers a person who has reached the first birthday but not the fifth, so a child listed as age 5 has aged out of the child category entirely and never reaches the 185%-of-poverty income screen. Income is not the discriminator in this household: on the identical ~$30,915 of income, the age-1 child is WIC-eligible while the age-5 child is not. Every wrong model treated ""age 5"" as inside the range — several stating the correct ""under 5"" rule and then reversing it, others stretching the boundary to ""through age 5"" — and then reinforced the wrong answer with an income or adjunctive-eligibility finding that cannot cure a categorical failure. Adjunctive eligibility through SNAP, Medicaid, TANF, or housing assistance substitutes only for the income test, never for the age category."
+us,scenario_008,child5_wic_eligible,13,llm_error,age_disability,False,,"WIC's child category covers children from age 1 up to their 5th birthday, so ages 1 through 4. A child whose age is listed as 5 has already turned 5 and has aged out. Income does not decide this case: the age-1 Child 6 is WIC-eligible in the same household, so the household meets the income test. All 13 models treated age 5 as still inside the WIC age window, reading 'under 5' or 'up to the 5th birthday' as if it included age 5."
us,scenario_008,child6_chip_eligible,12,llm_error,health_coverage,False,,"CHIP is residual coverage: PolicyEngine returns is_chip_eligible False for any child who already qualifies for Medicaid. The 1-year-old in this eight-person New Jersey household, with income near $30,915 (roughly 54% FPL), falls squarely inside the YOUNG_CHILD Medicaid category, so the Medicaid flag is 1 and the CHIP flag is 0. All twelve models tested only the upper NJ FamilyCare CHIP ceiling (300–355% FPL), found the household far beneath it, and treated that as proof of CHIP eligibility — when clearing that ceiling by such a margin is exactly what places the child under the lower young-child Medicaid threshold. Several went further and merged Medicaid and CHIP into a single ""NJ FamilyCare"" eligibility, erasing the mutual exclusivity that decides this output."
us,scenario_008,child6_early_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Early Head Start serves infants and toddlers under age 3, so age 1 satisfies the program’s age requirement. The household’s approximately $30,915 annual income is below the applicable 100% federal poverty line for this eight-person household, making Child 6 eligible."
us,scenario_008,child6_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Head Start requires both preschool-age categorical eligibility and satisfaction of the program's income or another qualifying eligibility pathway. Child 6 is age 1 and therefore belongs in the Early Head Start age category, not the preschool-age Head Start category; the household also exceeds the applicable income cutoff."
us,scenario_008,child6_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid applies a YOUNG_CHILD eligibility pathway to the 1-year-old dependent. At a MAGI income level of 0.55 times FPL, the child satisfies that category's income test and is eligible without any additional benchmark-triggering fact."
us,scenario_008,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_008,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"The reference adds two pieces. The first is the 2026 EITC maximum for three or more children, $8,231, with no reduction because AGI of about $30,632 is below the joint-filer phase-out start. The second is the refundable CTC: 15% x ($30,517.41 - $2,500) = $4,202.61 for the five children under 17. Earned income is wages plus self-employment income minus the half-SE-tax deduction ($4,000 - $282.59). Models went wrong in three recurring ways: a stale, inflated or phased-out EITC; earned income taken as gross $30,800 or as 0.9235 x SE income; and the $3,000 ACTC threshold instead of $2,500. A few also added a refundable AOTC even though qualified education expenses are $0."
+us,scenario_008,federal_refundable_credits,42,llm_error,thresholds_rates,False,,"The reference has two parts. The first is the full 2026 three-or-more-child EITC maximum of $8,231, with no phase-out because AGI (about $30,632) is below the 2026 MFJ phase-out threshold. The second is a refundable CTC of 15% × (earned income − $2,500). Earned income here is $26,800 wages plus $4,000 SE income minus the $282.59 half-SE-tax deduction, or $30,517.41, so the refundable CTC is $4,202.61. Only five children count for the CTC because the 17-year-old is excluded. Five times the $1,700 refundable cap is not binding, tax liability is zero, and the AOTC is $0 because the household has no qualified expenses. Most models missed on at least one of these steps: they used the wrong EITC maximum or applied a phase-out that does not apply, used the expired pre-TCJA $3,000 ACTC threshold, used gross $30,800 or 0.9235×SE earned income, or added an AOTC with no expenses."
us,scenario_008,free_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Free school meals do not require an explicitly listed direct-certification indicator in the household inputs. The household qualifies through both routes established by the computation: its school-meal income is 55% of the federal poverty guideline, below the 130% free-tier threshold, and all household members are SNAP-eligible, which supplies categorical eligibility."
-us,scenario_008,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA Medicaid expansion, so any non-elderly adult (19-64) with MAGI at or below 138% FPL for their household size qualifies through the MAGI-based ADULT category — no disability, pregnancy, or caretaker status required. This household of eight has total countable MAGI income of $30,915 ($4,000 self-employment + $15 interest + $26,800 wages + $100 interest), which is 0.55 x the eight-person FPL (roughly $55,000 in 2026), far under the 138% line of roughly $76,000. Every wrong model either compared that income to a threshold sized for a much smaller household, demanded a pre-ACA categorical hook, refused to compute eligibility absent an explicit input flag, or invented a housing-assistance disqualification that does not exist in MAGI Medicaid."
+us,scenario_008,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"New Jersey expanded Medicaid under the ACA. Any non-elderly adult with household MAGI at or below 138% FPL qualifies in the ADULT category, with no disability requirement. This household has 8 members. Its MAGI is about $30.8k: $26,800 in wages, plus $4,000 in self-employment income minus half of SE tax, plus $115 in interest. That works out to 0.55× FPL, well under the 138% limit, so the 42-year-old head is eligible. Every model answered No for one of three reasons: it assumed non-disabled adults are categorically excluded, it misjudged where the income falls against the threshold, or it made up an exclusion rule."
us,scenario_008,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not conferred on every household member by the presence of an eligible infant or young child. The 42-year-old Head is not pregnant, postpartum, or breastfeeding and is not an infant or child under age 5, so satisfying the household income limit does not make the Head eligible."
us,scenario_008,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_008,payroll_tax,46,reference_engine_defect,payroll_tax_base,False,,"Federal employee FICA on the $26,800 of wages is $2,050.20, and nearly every model got that part right. What separates the answers is New Jersey's mandatory employee payroll tax. The reference adds NJ temporary disability insurance at 0.19% ($50.92) and family leave insurance at 0.23% ($61.64), for $112.56. It counts no NJ unemployment, workforce or supplemental workforce fund employee contribution, although New Jersey workers pay 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds; the exclusion's corrected value counts them ($113.90 more), for $2,276.66. Most models either stopped at FICA ($2,050.20) or added the 0.3825–0.425% UI/WF/SWF contribution, often with outdated or inflated TDI/FLI rates (0% TDI, 0.09% FLI, or 0.33% FLI). gpt-6-astra and gpt-6-sol match the exclusion's corrected value, $2,276.66. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (payroll_tax_base). The frozen reference is 2,162.76; the corrected value is 2,276.66. PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax. Reference is an engine defect; output excluded from scoring (N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates)."
+us,scenario_008,payroll_tax,45,llm_error,state_local_rule,False,,"Federal employee FICA on the spouse's $26,800 of wages is $2,050.20. Almost every model got that part right. The models differ on New Jersey's four mandatory employee contributions, all charged on the full $26,800 at 2026 rates: TDI 0.19% ($50.92), FLI 0.23% ($61.64), UI 0.3825% ($102.51) and workforce/SWF 0.0425% ($11.39), which add $226.46. Most models left out the NJ contributions entirely. Others used the 2024 rates (TDI 0%, FLI 0.09%) or the 2025 rates (TDI 0.23%, FLI 0.33%), or dropped one of the four components."
us,scenario_008,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a generic indicator that income is below the 185% FPL ceiling. This household is at 55% of the federal poverty guideline and also meets categorical eligibility, so it receives the superior free-meals tier and reduced_price_school_meals_eligible equals 0."
us,scenario_008,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax is 15.3% of 92.35% of the $4,000 net self-employment income: $4,000 × 0.9235 × 0.153 = $565.18. The deduction for one-half of self-employment tax is an income-tax deduction and does not reduce the self-employment tax liability itself."
-us,scenario_008,snap,45,llm_error,thresholds_rates,False,,"The correct benefit takes $2,576.25 in monthly gross income and subtracts the $513.33 earned income deduction and the FY2026 standard deduction of $299 for households of six or more. That leaves net income of $1,763.92, which rounds to $1,764. The 30% contribution is rounded up to $530, and the FY2026 maximum allotment for eight people is $1,789, so the benefit is $1,259 a month, or $15,108 a year. No shelter deduction applies: rent of $579 a month is below half of adjusted income, and no utility costs are listed. No medical deduction applies either: the spouse's over-the-counter costs are $25 a month, under the $35 threshold. NJ broad-based categorical eligibility waives the asset test. The wrong answers fall into four groups: stale or invented parameters (the FY2025 $1,756 maximum, standard deductions of $198 to $291); skipping the round-up of net income and the 30% contribution ($1,259.82 or $1,260 instead of $1,259); subtracting a monthly standard deduction once from annual income; and zeroing SNAP through an asset test, housing assistance or a wrong gross-income limit."
+us,scenario_008,snap,46,llm_error,thresholds_rates,False,,"New Jersey's broad-based categorical eligibility makes the household eligible, so the spouse's $69,500 in bank assets does not count. The benefit then follows from FY2026 parameters. Gross income of $2,576.25 minus the $299 standard deduction (6+ people) and the $513.33 earned income deduction gives $1,763.92, which rounds to $1,764. Thirty percent of that is $529.20, rounded up to $530, and the $1,789 maximum for eight people minus $530 gives $1,259 a month, or $15,108 a year. There is no shelter, utility, or medical deduction. Most wrong answers either used outdated or invented maximum allotments and standard deductions, or skipped the rounding of net income and the 30% contribution and landed at about $1,260 a month. The $0 answers wrongly applied an asset test or income test."
us,scenario_008,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA adult expansion, so a non-elderly adult 19-64 who is not a tax-unit dependent qualifies on the MAGI pathway at or below 138% FPL, with no categorical hook, no asset test, and no benefit-receipt disqualifier. The test compares household MAGI to the FPL for the full household size: roughly $30,900 of wages, self-employment income and interest against the eight-person FPL is 0.55 x FPL, less than half the 1.38 limit. Housing assistance is excluded from MAGI and the $69,500 in bank accounts is irrelevant because MAGI categories carry no resource test. Each wrong answer either skipped this income test entirely or overrode it with a disqualifier that does not exist."
us,scenario_008,spouse_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility here turns on a single age test: the person must be 65 or older, and the Spouse is 41. The trap is the `is_disabled` flag: every wrong model converted it into a disability-based Medicare pathway, but real-world under-65 Medicare entitlement runs through 24 months of SSDI cash benefits (or ESRD/ALS), none of which is present — the household reports no Social Security disability income, and the prompt directs that unlisted income and status inputs be treated as 0/false. A disability flag alone establishes neither SSDI entitlement nor the 24-month qualifying period, so it cannot produce Medicare eligibility for a 41-year-old."
us,scenario_008,spouse_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific: an adult must be pregnant, breastfeeding, or within the applicable postpartum period, while infants and children qualify only through their own age category. Low household income and the presence of a young child do not make the child's mother or caretaker independently eligible."
us,scenario_008,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_008,state_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"NJ gross income is $30,915. That is wages plus Schedule C profit plus interest, with no federal half-SE-tax adjustment and no standard deduction. Exemptions total exactly $12,000: $1,000 for each spouse, one $1,000 blind-or-disabled exemption for the spouse (a person who is both blind and disabled still gets only one), and 6 × $1,500 for the dependents. That leaves $18,915 taxable, and 1.4% of that is $264.81. The models went wrong in four recurring ways. Some counted blind and disabled as two exemptions or valued the personal and dependent exemptions wrong. Some deducted over-the-counter health expenses, which don't qualify for NJ's medical deduction. Some took the 18%-of-rent property tax deduction. Some subtracted the $50 property tax credit. That credit is refundable and is worth more than the deduction's $17.51 tax saving, so it is claimed below this line and the rent deduction isn't taken. A final group invented a standard deduction, credits or a low-income phase-out that drove the tax to zero."
-us,scenario_008,state_refundable_credits,46,llm_error,state_local_rule,False,,"The reference has three parts. The NJ EITC is 40% of the full $8,231 federal 3+-child EITC, or $3,292.40, because this married-filing-jointly household is below the 2026 federal phase-out start. The NJ Child Tax Credit is $2,500 for the two children aged 5 and 1, under the schedule New Jersey enacted June 30, 2026 for tax years 2026–2028. That schedule's tiers use NJ taxable income, which is about $18,900 after $12,000 of exemptions, not gross income of about $30,900. The last part is the $50 refundable property tax credit for tenants, since $6,949 of rent is listed. Most models used the old $1,000-per-child schedule, or wrongly put the family in the $800 or $400 tier based on gross income. Many also left out the $50 property tax credit, phased out the federal EITC by mistake, or said NJ has no refundable credits. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
+us,scenario_008,state_income_tax_before_refundable_credits,41,llm_error,state_local_rule,False,,"New Jersey taxable income here is NJ gross income ($30,915 = wages + full $4,000 SE profit + $115 interest, with no half-SE-tax adjustment) minus exemptions only. The exemptions are $1,000 for each spouse, $1,500 for each of the 6 dependents, and a single $1,000 blind-or-disabled exemption for the spouse, who gets only one even though they are both blind and disabled. That leaves $18,915 × 1.4% = $264.81. Models mostly went wrong in four ways: they claimed separate blind and disabled exemptions, used the wrong exemption amounts, subtracted the $50 property-tax credit (it is refundable, so it doesn't belong in this output) or the 18%-of-rent deduction (not taken, because the household takes the credit), or treated OTC health expenses as a deductible medical expense. Others zeroed the tax using a standard deduction or filing-threshold rule that New Jersey doesn't have."
+us,scenario_008,state_refundable_credits,47,llm_error,state_local_rule,False,,"The reference adds three New Jersey credits. The first is a $3,292.40 NJ EITC, equal to 40% of the full $8,231 federal maximum for 3+ children; at about $30.5k of earned income, the couple has not reached the MFJ phase-out. The second is a $2,500 NJ child tax credit for the two children age 5 and under, using the schedule enacted June 30, 2026 and keyed to NJ taxable income (well under $30,000 after exemptions), not gross income. The third is a $50 property tax credit for the renter household. Correctly applying the older $1,000-per-child CTC schedule gives only $5,342.40, so even flawless pre-enactment knowledge misses by $500. Most models also made their own errors on top of this: they phased out the federal EITC too early, chose the CTC tier from gross income, used outdated EITC maximums, left out the $50 credit, or denied that NJ has refundable credits at all. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned reference_later_law; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
us,scenario_008,tanf,1,llm_error,thresholds_rates,False,,"New Jersey's WorkFirst NJ TANF is not a residual top-up: it is gated on countable income falling below the family-size eligibility standard tied to the maximum grant (roughly $770/month for a household of eight) and on countable resources at or below the $2,000 limit. This household's $26,800 in wages plus $4,000 of self-employment income is about $2,576/month, which clears the standard by a wide margin even after New Jersey's 50% earned-income disregard, and the $69,500 bank balance independently fails the resource test. Six children plus a blind and disabled spouse make the household read as categorically needy, which is the trap; family size raises the grant schedule but nowhere near enough to admit income at this level. The grant is therefore $0."
-us,scenario_009,federal_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"The correct path is: AGI of $105,900 ($60,000 gross wages, $20,400 taxable pension, and $25,500 of Social Security, the 85% cap), minus the 2026 MFJ standard deduction of $32,200, gives taxable income of $73,700. Under the 2026 brackets (10% to $24,800, 12% to $100,800), with no nonrefundable credits, tax is $2,480 + $5,868 = $8,348. The wrong answers fall into four groups. Some models assumed TCJA expired in 2026 and applied personal exemptions, a roughly $16,000 standard deduction and a 15% bracket, landing near $10,500. Others inflated the pre-OBBBA $30,000 standard deduction and the 2025 brackets instead of using $32,200 and $24,800, landing near $8,500. A third group subtracted the spouse's $8,139 employer-sponsored insurance premium from gross wages. Finally, several Claude models derived about $8,300-$8,349 and then submitted an unrelated number."
+us,scenario_009,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"AGI is $105,900. That is $60,000 of gross wages, plus $20,400 of taxable pension, plus $25,500 of taxable Social Security, which is the 85% cap because provisional income of $95,400 is far above $44,000. The answer depends on using the OBBBA-permanent TCJA parameters for 2026: a $32,200 MFJ standard deduction, a 10% bracket up to $24,800 and 12% above that. Taxable income of $73,700 then gives $2,480 + $5,868 = $8,348. Most models went wrong in one of three ways: they assumed TCJA expired (personal exemptions and a 15% bracket), they used inflation-projected pre-OBBBA figures ($30,000–$30,900 standard deduction, $23,850–$24,500 bracket), or they subtracted the ESI premium from wages, which are already given as gross W-2 income. Several models worked out $8,348 or close to it and then submitted an unrelated number."
us,scenario_009,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_009,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable under-65 pathway such as qualifying disability, ESRD, or ALS. Both adults are 57, and the prompt makes every unlisted disability or health status false, so the listed Social Security retirement income cannot be reclassified as evidence of disability-based Medicare eligibility."
us,scenario_009,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_009,payroll_tax,7,llm_error,payroll_tax_base,False,,"The $8,139 employer-sponsored insurance premium is not an employee pre-tax payroll deduction and therefore does not reduce the spouse's FICA wage base. Applying the employee rates of 6.2% Social Security and 1.45% Medicare to the full $60,000 of wages yields $3,720 plus $870, totaling $4,590."
+us,scenario_009,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 6.2% Social Security plus 1.45% Medicare on the spouse's full $60,000 gross wages: $3,720 + $870 = $4,590. No Additional Medicare Tax applies, and NC has no mandatory employee state payroll tax. The household facts list no Section 125 cafeteria-plan salary reduction. The ESI premium field is a premium amount and not a payroll deduction, which is clear because the head has $8,089 of ESI premiums with no wages at all. Most models invented an unstated pre-tax exclusion and taxed $51,861 instead of $60,000. Two others used the wrong rates or added an amount with no basis."
us,scenario_009,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_009,spouse_medicare_eligible,1,llm_error,other,False,,"Medicare eligibility is absent because the spouse is age 57, below the standard eligibility age of 65, and no alternative eligibility status was listed. The required binary encoding assigns 0 to “No” and 1 to “Yes,” so the spouse’s result is 0."
us,scenario_009,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_009,state_income_tax_before_refundable_credits,26,llm_error,taxable_income_or_deductions,False,,"The reference result comes from four steps. Federal AGI is $105,900: the spouse's full $60,000 gross wages, plus the $20,400 taxable pension, plus $25,500 taxable Social Security. North Carolina then subtracts the $25,500 of Social Security and the $25,500 married-filing-jointly standard deduction, which leaves $54,900 of taxable income. The 2026 flat rate of 3.99% on $54,900 gives $2,190.51, and no nonrefundable credits apply. The wrong answers split into two main groups. The first group cut the spouse's wages by the $8,139 employer-sponsored insurance premium, giving an AGI of $97,761 and taxable income of $46,761; some of these also imputed itemized deductions. The second group used a stale flat rate (4.25%, 4.5%) or a wrong standard deduction ($21,000 to $30,000). A few models computed $2,190.51 correctly and then submitted a different number."
+us,scenario_009,state_income_tax_before_refundable_credits,27,llm_error,taxable_income_or_deductions,False,,"North Carolina's tax starts from federal AGI of $105,900: $60,000 in gross wages, $20,400 of taxable pension and $25,500 of taxable Social Security. The listed employer-sponsored insurance premiums do not reduce wages. NC then subtracts the $25,500 of Social Security included in federal AGI and its fixed, non-indexed $25,500 MFJ standard deduction. That leaves $54,900, taxed at the 2026 flat rate of 3.99% for $2,190.51, with no nonrefundable credits. The wrong answers fall into four groups. One subtracted the $8,139 ESI premium from wages, giving $46,761 and $1,865.76. Another used an old rate (4.25% or 4.5%) or the wrong standard deduction. A third invented itemized mortgage interest or NC credits. The last computed $2,190 and then submitted a different number."
us,scenario_009,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_009,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_012,child1_chip_eligible,9,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying CHIP age and income limits alone is not sufficient. The 10-year-old qualifies for Medicaid under Mississippi's OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility."
+us,scenario_012,child1_chip_eligible,10,llm_error,categorical_eligibility,False,,"The household has $22,000 of income, about 83% of FPL for a family of 3 (about 68% for a family of 4). That is far below Mississippi's Medicaid limit for children aged 6-18, which is 133% FPL plus the 5-point disregard. The 10-year-old is therefore Medicaid-eligible in the OLDER_CHILD category. CHIP only covers children who do not qualify for Medicaid, so Medicaid eligibility rules out CHIP. Every model tested the child only against the higher CHIP income ceiling and answered Yes. None of them applied the rule that Medicaid eligibility comes first."
+us,scenario_012,child1_head_start_eligible,1,parse_contract_failure,other,False,,"Head Start eligibility depends on age, and the deciding fact is that Child 1 is 10, well above the preschool range of 3 to 4. The household's income is below the poverty guideline and it gets SNAP, so it meets the income and categorical tests, but those don't matter once the child fails the age test. The model's reasoning identified the age bar and ended with 'value = 0', yet its recorded answer was Yes (1). The error is therefore a mismatch between the submitted value and the explanation, not a mistake in reasoning about the policy."
us,scenario_012,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi evaluates a 10-year-old dependent through the Medicaid older-child MAGI category. Household MAGI is 0.81 times FPL, below the applicable eligibility threshold, so the category and income tests produce eligibility."
-us,scenario_012,child1_wic_eligible,3,llm_error,categorical_eligibility,False,,WIC child eligibility is limited to children under age five; low household income does not extend eligibility to older children. Child 1 is age 10 and therefore fails the categorical age requirement before the income test can establish eligibility.
+us,scenario_012,child1_wic_eligible,4,llm_error,age_disability,False,,"To get WIC, a person must fit a category: pregnant, postpartum or breastfeeding woman, infant, or child under 5. In federal rules, the child category ends at the 5th birthday. Child 1 is 10, so the child fails the categorical test no matter how low the household income is, which makes child1_wic_eligible No. Every wrong model let low income decide the answer, or used the wrong age cap, or wrote that the child failed but still submitted 1."
us,scenario_012,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_012,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household files jointly with one qualifying child (age 10). $22,000 of earnings sits on the one-child EITC plateau, which is past the phase-in end of about $13,000 and below the MFJ phase-out start, so the full 2026 maximum of $4,427 applies. Tax liability is zero because $22,000 is below the MFJ standard deduction, so the OBBBA-permanent $2,200 CTC is refundable only up to the 2026 ACTC cap of $1,700. The 15% × ($22,000 − $2,500) = $2,925 earnings test does not bind. Most wrong answers used the wrong parameter year or regime. They used the 2025 EITC maximum ($4,328), a hand-inflated EITC, the TCJA-sunset ACTC ($1,000 cap above $3,000), a $1,800 or $2,000+ refundable cap, or counted the unborn child or the pregnant spouse as a qualifying child."
+us,scenario_012,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"This is a married couple filing jointly with one qualifying child (age 10) and $22,000 of earnings. That income is above the end of the one-child EITC phase-in and below the joint phase-out start, so the household gets the full 2026 one-child maximum of $4,427. Federal income tax is $0. Under the OBBBA-permanent CTC rules, the refundable ACTC is min($1,700 refundable cap, 15% × ($22,000 − $2,500) = $2,925) = $1,700, for a total of $6,127. Most wrong answers either used the 2025 EITC maximum ($4,328) or another un-indexed figure, or got the ACTC wrong in one of three ways: the TCJA-sunset $1,000 credit, the full $2,000/$2,200 credit treated as refundable, or $1,800. A few also miscounted children by including the pregnancy, or applied phase-outs that do not reach $22,000."
us,scenario_012,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"This three-person Mississippi household has $22,000 in wages, which is 81% of the federal poverty guideline. That is far below the 130% free-meal cutoff. The household also receives about $407.90 a month in SNAP, which makes the 10-year-old categorically eligible for free meals, and PolicyEngine treats a 10-year-old as enrolled in K-12. So the engine puts the household in the FREE tier. The wrong answers came from three different errors: one model never ran the income or SNAP test, one treated the child's school enrollment as an unlisted input set to false, and one wrongly claimed income was above 130% of the poverty guideline."
us,scenario_012,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Mississippi Medicaid eligibility for a nondisabled 40-year-old adult requires qualification through a defined categorical pathway; income below 138% FPL does not independently establish eligibility. The head’s MAGI is 0.81 FPL, but the Medicaid category is NONE because the head satisfies none of Mississippi’s available pathways."
us,scenario_012,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not extended to every member of a household containing a pregnant participant. The pregnant spouse satisfies the categorical requirement, but the 40-year-old head is neither pregnant, postpartum, breastfeeding, an infant, nor a child under age five, so the head is not WIC-eligible despite household income below the limit."
us,scenario_012,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_012,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_012,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. At a school-meal FPG ratio of 0.81 and with categorical eligibility, the household is classified as FREE, so it receives no reduced-price support."
-us,scenario_012,snap,45,llm_error,taxable_income_or_deductions,False,,"This is a three-person SNAP unit. The unborn child does not count as a member. Gross income is $1,833.33 a month. Subtracting the $209 standard deduction and the $366.67 earned income deduction (20% of earnings) gives net income of $1,258 after rounding. Under the 2026 rule (P.L. 119-21), receiving an energy (LIHEAP) subsidy gives a Standard Utility Allowance only to households with an elderly or disabled member, so this household gets no utility allowance. Its $450 rent is below half of net income, so there is no excess shelter deduction. Thirty percent of net income, rounded up, is $378, and $785 − $378 = $407 a month, or $4,884 a year. Most wrong answers either granted a LIHEAP-triggered utility allowance and shelter deduction, counted the unborn child to make a household of four, used outdated standard-deduction or maximum-allotment figures, or skipped SNAP's whole-dollar rounding (30% of net income rounded up, allotment rounded down) and got $4,892–$4,896."
+us,scenario_012,snap,46,llm_error,taxable_income_or_deductions,False,,"The SNAP unit has three people. Pregnancy does not add a member. Monthly net income is $1,833.33 − $209 standard deduction − $366.67 earned-income deduction = $1,258. Shelter cost is only the $450 rent. The $411 energy subsidy does not give this household, which has no elderly or disabled member, a standard utility allowance under the 2026 rules, so there is no excess shelter deduction because $450 is less than half of net income ($629). The benefit is $785 minus 30% of net income rounded up to $378, which is $407 a month or $4,884 a year. Most wrong answers either added a utility allowance triggered by the energy subsidy, counted the unborn child as a fourth member, used older FY2025 deduction and allotment amounts ($204/$768), or skipped SNAP's rounding rules ($407.70 instead of $407)."
us,scenario_012,spouse_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person not already qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under Mississippi's OLDER_CHILD category, so that Medicaid eligibility precludes CHIP regardless of pregnancy status or how far household income falls below a CHIP-related threshold."
us,scenario_012,spouse_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi Medicaid applies an OLDER_CHILD eligibility pathway to the 18-year-old spouse. The spouse's MAGI of 0.67 times FPL is below the applicable income threshold, so this category produces eligibility despite the spouse not being a tax-unit dependent."
us,scenario_012,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC covers pregnant individuals who satisfy the program's income test. The spouse is pregnant, and the household's approximately $22,000 annual income is below the applicable 185% federal-poverty income limit, so the spouse meets both categorical and financial eligibility requirements."
@@ -106,21 +107,22 @@ us,scenario_012,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_012,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"Mississippi taxable income for this married-joint household is $22,000 minus the $4,600 MFJ standard deduction, the $12,000 MFJ personal exemption, and the $1,500 exemption for the one dependent child, which leaves $3,900. Mississippi taxes the first $10,000 of taxable income at 0% and applies its flat rate (4.0% in 2026) only above $10,000, so the joint liability is $0. PolicyEngine's per-spouse allocation gives the head $118 (4% of $12,950 minus $10,000), but the unit-level result is $0. Four models computed the $3,900 base correctly and then taxed it from the first dollar. The rest also got the deductions or exemptions wrong, and one of them used another state's credit structure."
us,scenario_013,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The trap is the Social Security provisional-income test: half of the $23,736 benefit ($11,868) plus the $6,720 pension, $16 of dividends and $72 of capital gain gives provisional income of $18,676, below the $25,000 first base amount for a single filer, so none of the Social Security is includible. AGI is therefore $6,808, which is under the single standard deduction before any age-65 additional amount or the OBBBA senior deduction is even reached, leaving taxable income and tax of $0. A second, independent guardrail is that federal income tax before refundable credits is floored at zero — nonrefundable credits can only zero out liability, never drive it negative."
us,scenario_013,head_medicaid_eligible,3,llm_error,age_disability,False,,"The head is 80, so the 138% FPL MAGI adult expansion group (ages 19-64) is closed to them. The only route left is the aged/blind/disabled pathway, which counts Social Security in full. Social Security of $23,736 plus the $6,720 pension and $88 of investment income comes to about $30,544, or 1.91x FPL. That is far above the SSI benefit rate and the 100% FPL aged/disabled standard, and with SSI at 0 there is no SSI-linked eligibility either, so the engine returns category NONE. All three models answered Yes. One asserted eligibility without testing income, one excluded most of the Social Security, and one used AGI, which leaves out the nontaxable Social Security."
-us,scenario_013,snap,46,llm_error,categorical_eligibility,False,,"Countable gross income is $2,539.33 a month. That is Social Security, pension and dividends; the $72 of capital gains is not counted. From March 2026, Arizona's expanded categorical eligibility limit is 200% FPL: $2,608.33 a month, rising to $2,660 from October. Because income is under that limit, the household is categorically eligible, and the asset test and net income test are treated as met. A categorically eligible one-person household gets the $24 minimum allotment even when $298 minus 30% of net income is below zero, so benefits are $24 × 10 months (March–December) = $240. Every model either applied the regular asset or net income tests, used the old 185% limit, or stopped when the benefit formula came out at zero, and never applied the minimum benefit to a categorically eligible household."
+us,scenario_013,snap,47,llm_error,categorical_eligibility,False,,"Starting in March 2026, Arizona's broad-based categorical eligibility (BBCE) covers households with gross income up to 200% FPL. That limit is $2,608.33/month, rising to $2,660 from October. This household's gross income is $2,539.33/month (Social Security, pension and dividends), which is under that limit. BBCE treats both the asset test and the net income test as met, so the $58,700 in the bank and the $2,138 net income no longer disqualify it. The benefit formula ($298 minus 30% of net income) comes out below zero, but a categorically eligible one-person household still gets the $24 monthly minimum. That gives $24 × 10 months (March–December) = $240. Every model answered $0. They either applied the regular asset and net income tests, used Arizona's old 185% BBCE limit, or treated a negative benefit formula as ineligibility instead of applying the minimum benefit."
us,scenario_013,state_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Arizona’s standard deduction eliminates the remaining taxable income after excluding Social Security, so applying the 2.5% rate directly to gross dividends and capital gains is incorrect. The resulting Arizona income tax before refundable credits is zero."
-us,scenario_013,state_refundable_credits,36,llm_error,state_local_rule,False,,"The entire $25 is Arizona's refundable Credit for Increased Excise Taxes (A.R.S. 43-1072.01). It pays $25 per person in the tax unit (taxpayer, spouse and dependents, up to $100) when AGI is at or below $12,500 for a single filer. Because it is refundable, it does not depend on tax liability, earned income, dependents, rent or property tax. This single filer's AGI is $6,736, since Social Security is excluded, so the tax unit of one gets $25. Almost every model left this credit out and assumed Arizona's refundable credits require dependents, earnings or property-tax/rent payments. Of the rest, two tested the wrong income, two misread the credit's conditions, and one multiplied the $25 by age and disability exemptions."
-us,scenario_014,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The only taxable income is the spouse's $88,927.65 of gross wages. Veterans benefits are excluded, and the listed ESI premium does not reduce the stated gross wages. Under 2026 law as amended by OBBBA, the TCJA structure is permanent: the MFJ standard deduction is $32,200, personal exemptions are zero, and the brackets are 10% to $24,800 and 12% to $100,800. That gives taxable income of $56,727.65 and tax of $2,480 + 12% x $31,927.65 = $6,311.32, with no nonrefundable credits. Most wrong answers came from assuming the TCJA sunset (pre-TCJA deduction, personal exemptions, 15% bracket), using 2025 or guessed deduction and bracket figures, subtracting the $14,717 premium from wages, or replacing a correct calculation with a different number."
+us,scenario_013,state_refundable_credits,37,llm_error,state_local_rule,False,,"Arizona's Credit for Increased Excise Taxes (A.R.S. §43-1072.01) is refundable and pays $25 for the taxpayer plus $25 for each dependent, capped at $100. The single-filer limit is $12,500 of income. This person's federal AGI is $6,736 because none of the Social Security is taxable at their income level, so one $25 credit applies. They don't need earned income, dependents, tax liability, rent, or property tax to get it. Nearly every model missed this credit, ruled it out with made-up requirements, or misread the income test. One model counted age and disability exemptions as extra credit units."
+us,scenario_014,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The only taxable income is the spouse's $88,927.65 in wages, and AGI equals that full amount. Veterans benefits are excluded, and the listed ESI premium is not subtracted from the gross wage input. Under OBBBA's permanent 2026 parameters, the MFJ standard deduction of $32,200 leaves taxable income of $56,727.65. Tax is then 10% of the first $24,800 plus 12% of the remaining $31,927.65, which equals $6,311.32, and no nonrefundable credits apply because there are no dependents. Most models went wrong in one of four ways: they assumed the TCJA had sunset (bringing back personal exemptions and a 15% bracket), used stale 2025 deduction and bracket figures, subtracted the ESI premiums from wages, or did the math correctly and then submitted a different number."
us,scenario_014,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"Federal refundable credits are zero because this childless joint tax unit has income far above the EITC phase-out range and has no qualifying child for refundable CTC. No education, rebate, or refundable payroll-tax-credit facts create another refundable federal credit."
us,scenario_014,head_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"Medicaid eligibility for the head uses household tax-unit MAGI rather than only the head's earned income. The household's MAGI is approximately $183,584, or 4.11 times FPL, exceeding every applicable West Virginia Medicaid income limit, and the head has no alternative categorical pathway."
us,scenario_014,payroll_tax,16,llm_error,payroll_tax_base,False,,"Only the spouse has wages, so the correct answer is the 7.65% employee FICA rate applied to her full $88,928 gross wages: $5,513.51 Social Security plus $1,289.45 Medicare, or $6,802.96. The facts do not say she pays premiums through a salary-reduction (Section 125) plan, so the $14,717 in employer-sponsored insurance premiums does not come out of the FICA wage base. Veterans benefits are not wages, and West Virginia has no employee state payroll tax. The largest group of wrong answers (7 models) assumed a cafeteria plan that the facts never mention and taxed only $74,211. Most of the rest applied 7.65% to $88,928 and then invented an extra tax, made an arithmetic slip, or submitted a number different from their own total."
-us,scenario_014,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"West Virginia AGI is the spouse's full $88,927.65 of wages; veterans benefits are exempt, and the listed ESI premium does not reduce the listed gross wages. Subtracting two $2,000 exemptions leaves $84,927.65 of taxable income, with no standard deduction. For 2026, WV applies MFJ rates 5% below the 2025 schedule: 2.11% to $10k, 2.81% to $25k, 3.16% to $40k, 4.22% to $60k and 4.58% above. That gives 211 + 421.50 + 474 + 844 + 1,141.69 = $3,092.19. Most models used the correct base but either the 2025 rates (2.22–4.82%, giving $3,255.03) or the 2023–24 rates (2.36–5.12%, giving $3,459.81). The rest invented standard deductions, subtracted the ESI premium, or added credits that do not exist."
-us,scenario_015,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The correct path is AGI of $42,909 ($45,000 wages minus the $1,042 traditional 401(k) deferral, plus $2,000 interest, minus the $3,000 capital-loss cap and the $49 IRA deduction; workers' comp excluded). Subtract the 2026 single standard deduction of $16,100 to get $26,809 taxable income, then apply the 2026 brackets: 10% to $12,400 and 12% above, for $2,969.13. No nonrefundable credit applies, because the Saver's Credit and the elderly/disabled credit are both phased out at this AGI. Most models got AGI right and then missed on the 2026 parameters. Some used 2024/2025 standard deductions and bracket tops ($14,600, $15,000, $15,750; $11,600, $11,925). Others assumed the TCJA sunset (personal exemption plus a 15% bracket), even though the 2025 law made the TCJA rate and standard-deduction structure permanent. The rest fabricated itemized deductions (mortgage interest from the loan balance, double-counted premiums), excluded health premiums from wages, invented credits, or submitted a number that contradicts their own arithmetic."
+us,scenario_014,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"West Virginia AGI is the spouse's $88,928 of wages. The veterans benefits are excluded, and the employer-sponsored insurance premium input does not reduce wages. WV has no standard deduction, so the only subtraction is two $2,000 personal exemptions, which leaves $84,927.65 of taxable income. The deciding step is the 2026 MFJ rate schedule. It is the 2025 schedule (2.22/2.96/3.33/4.44/4.82%) cut by another 5%, to about 2.11/2.81/3.16/4.22/4.58%, which yields $3,092.19 rather than $3,255 (2025 rates) or $3,460 (2024 rates). Many models also changed the base: they subtracted the $14,717 ESI premium to get $74,211, made up a standard deduction, or applied invented credits."
+us,scenario_015,federal_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"The correct calculation is short. Start with wages of $45,000 minus the $1,042 traditional 401(k) deferral, add $2,000 of interest, subtract the $3,000 capped capital loss and the $49 IRA deduction, and AGI is $42,909. Subtract the $16,100 single standard deduction for 2026 (OBBBA made the TCJA structure permanent, so there is no personal exemption) to get taxable income of $26,809. Tax at 10% up to $12,400 and 12% above that is $1,240 + $1,729.13 = $2,969.13. No nonrefundable credits apply: there are no dependents, AGI is above the Saver's Credit limit, and the elderly/disabled credit is phased out. Most models got AGI right and then missed the answer in one of four ways: they used 2024 or 2025 standard deductions and bracket edges ($14,600, $15,000 or $15,750; $11,600, $11,925 or $12,150), assumed the TCJA had expired and applied a personal exemption with a 15% bracket, made up deductions such as mortgage interest or pre-tax health premiums, or made arithmetic and transcription errors after reaching about $2,969."
us,scenario_015,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A single adult with no dependent children or qualifying education expenses receives neither the refundable Child Tax Credit nor the refundable American Opportunity Credit. At approximately $43,450 of income, the childless EITC is also zero, and all other evaluated refundable-credit components are zero, yielding total federal refundable credits of $0."
us,scenario_015,head_chip_eligible,1,parse_contract_failure,other,False,,"CHIP eligibility runs through categorical gates first: coverage is for children under 19 and, at state option, pregnant women, so a 36-year-old head never reaches the income test — PolicyEngine assigns chip_category = 3 (categorically ineligible) and is_chip_eligible = False. The ~$43,450 household income and Indiana's Hoosier Healthwise/CHIP income thresholds are irrelevant to a person who fails the age/pregnancy gate, and the separate Medicaid pathways also return NONE for a non-aged, childless adult at that income in a state whose adult expansion he clears on neither disability-SSI nor MAGI grounds. The single wrong answer stated this categorical rule correctly in prose and then emitted the opposite binary."
us,scenario_015,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish Medicaid eligibility; the person must satisfy a specific Indiana eligibility pathway and its applicable requirements. The head falls into Medicaid category NONE, receives no SSI, is not a dependent, and has MAGI at 2.69 times FPL, so no MAGI, disability-related, or state-specific pathway confers eligibility."
us,scenario_015,head_medicare_eligible,4,llm_error,age_disability,False,,"A person under 65 becomes eligible for Medicare on disability grounds only after 24 months of entitlement to Social Security Disability Insurance, or through the ESRD or ALS pathways. Being disabled is not enough on its own. The head is 36, has no Social Security disability benefits, and has no ESRD or ALS status. That means they don't meet the age-65 test or any disability-entitlement pathway, so they are not Medicare eligible. All four models treated the bare 'is disabled' flag as automatic Medicare eligibility and skipped the SSDI-receipt requirement."
+us,scenario_015,local_income_tax,47,prompt_ambiguity,state_local_rule,False,,"The prompt describes local_income_tax by listing only NYC, Philadelphia, Kansas City and St. Louis taxes. PolicyEngine's local_income_tax variable also includes Indiana county income tax. No county was given, so the engine used Adams County and applied its 1.6% rate to Indiana AGI of $41,909.38: federal AGI of $42,909.38 (after the $3,000 capital-loss deduction and the $48.69 IRA deduction) minus the $1,000 exemption, which gives $670.55. Every model answered $0, either because it read the four-city list as the complete scope or because it wrongly said Indiana has no local income tax. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
us,scenario_015,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $45,000 of gross wages: 6.2% Social Security ($2,790, well under the 2026 wage base) plus 1.45% Medicare ($652.50) gives $3,442.50. Indiana has no mandatory employee state payroll tax, and wages are far below the $200,000 Additional Medicare Tax threshold. The most common wrong move was shrinking the FICA base. Four models treated the listed $2,080 health insurance premiums as a Section 125 pre-tax salary reduction, and one subtracted the traditional 401(k) deferral, even though elective 401(k) deferrals stay subject to FICA. The rest reached or approached $3,442.50 and then broke the total with arithmetic slips or made-up add-ons."
-us,scenario_015,state_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"Indiana starts from federal AGI of $42,909.38. That is wages less the $1,042 traditional 401(k) and the $49 IRA deduction, plus $2,000 of interest, less the capital loss capped at $3,000. Workers' compensation is excluded. Indiana then subtracts only the $1,000 personal exemption and applies the scheduled 2026 flat rate of 2.95%: $41,909.38 x 0.0295 = $1,236.33. Most models built this base correctly but used the wrong rate: 3.05% (2024), 3.0% (2025), 2.9%, 2.85% or a made-up rate. Others added deductions Indiana does not allow. These were an extra exemption for being disabled (Indiana's additional $1,000 exemptions cover age 65+ and blindness only), a deduction for the $2,080 in health premiums, a child support deduction, or a standard deduction."
+us,scenario_015,state_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"Indiana's 2026 tax is its flat 2.95% rate applied to federal AGI minus a single $1,000 personal exemption. Federal AGI is $42,909: wages of $45,000, minus the $1,042 traditional 401(k), minus the $49 IRA deduction, plus $2,000 interest, minus the $3,000 capped capital loss. Workers' compensation is excluded. That gives $41,909 × 2.95% = $1,236. Most models missed for one of three reasons. Some used the wrong 2026 rate (3.0%, 3.05%, 2.85%, 2.9%, 2.97%, 3.025%). Some invented exemptions or deductions that do not exist: a disability exemption for a non-blind taxpayer under 65, a health-premium deduction, a child-support deduction, or a standard deduction. Others built AGI incorrectly, for example by deducting the full $4,510 loss, including workers' comp, or skipping the 401(k) exclusion."
us,scenario_015,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,federal_income_tax_before_refundable_credits,8,llm_error,taxable_income_or_deductions,False,,"The household reports $47,000 of deductible medical outlays — $26,000 of health insurance premiums excluding Medicare Part B, $20,000 of other medical expenses, and $1,000 of over-the-counter health expenses — against an AGI of $32,137.94 (wages of $34,638 less the $2,500 statutory cap on the student loan interest deduction). Net of the 7.5%-of-AGI floor of $2,410, the itemized medical deduction of roughly $44,590 exceeds AGI outright, so itemizing dominates the ~$16,100 standard deduction, taxable income is zero, and federal income tax before refundable credits is $0. Seven of the eight models ran only the standard-deduction path and clustered in the $1,271–$1,873 band; the eighth recognized the medical deduction but reported the unused excess as a large negative liability instead of flooring the tax at zero."
us,scenario_016,federal_refundable_credits,3,llm_error,credit_phaseout,False,,"This is a single childless 49-year-old with $34,638 in wages, so the only refundable credit in play is the childless EITC, and its complete phase-out point for a single filer is roughly $19,100 of earned income/AGI in 2026 — barely more than half this filer's wages. With no qualifying children, no education expenses, no recovery rebate, and no excess payroll tax withholding, every component PolicyEngine evaluated (EITC, refundable AOTC, refundable CTC, Recovery Rebate Credit, refundable payroll tax credit) is $0. Both models that produced a number stopped at ""the childless EITC exists"" without applying the terminating phase-out threshold that zeroes the credit far below this income."
@@ -138,70 +140,70 @@ us,scenario_016,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_016,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_018,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"The correct path is simple: AGI of $61,592 (wages, less the traditional 401(k), plus $50 interest, less the $289 IRA deduction), minus the 2026 OBBBA standard deduction of $16,100, gives $45,492 of taxable income. The 2026 single brackets (10% to $12,400, then 12%) put the tax at $5,211.05, with no nonrefundable credits. Most wrong models used stale parameters: the 2024 or 2025 standard deduction ($14,600, $15,000, $15,400 or $15,750) and 10% bracket tops ($11,925 or $12,150), or they assumed TCJA had expired, bringing back personal exemptions and the 15% and 25% brackets. A second group took deductions that the listed facts do not support: an inferred $6,760 qualified overtime premium, a $1,625 car-loan interest deduction, or itemized medical deductions that counted the $9,000 premium twice. A few made arithmetic errors, submitted numbers that contradict their own work, or invented dependents or a saver's credit."
+us,scenario_018,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The calculation is simple. AGI is $61,592, the 2026 single standard deduction is $16,100, and taxable income of $45,492 is taxed at 10% up to $12,400 and 12% above that, giving $5,211.05. There are no nonrefundable credits. Most models missed because they used stale or invented parameters: the 2024/2025 standard deductions ($14,600, $15,000, $15,750), old 10% bracket ceilings ($11,925, $12,200, etc.), or a TCJA sunset with personal exemptions that OBBBA prevented. The rest added deductions that the facts don't support: a $6,760 overtime-premium deduction, a $1,625 car-loan interest deduction, double-counted medical premiums that pushed itemized deductions above the standard deduction, or Saver's/Child Tax Credits this single, childless filer with $61.6k AGI can't claim."
us,scenario_018,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_018,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $68,005 of wages here: $4,216.31 of Social Security tax plus $986.07 of Medicare tax equals $5,202.38. The listed health-insurance premiums were not identified as employee pre-tax salary-reduction contributions, and Arizona's lack of a separate state payroll tax does not remove federal employee FICA."
us,scenario_018,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_018,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The calculation has three steps: federal AGI of $61,592.09 (wages minus the traditional 401(k), plus interest, minus the traditional IRA deduction), then Arizona's 2026 single standard deduction of $15,750, then the flat 2.5% rate on $45,842.09, which gives $1,146.05. Itemizing does not win: Arizona itemized deductions are $10,786.71 ($9,500 medical plus $1,286.71 in state and local taxes), well below $15,750. Most models got AGI and the 2.5% rate right but used the wrong standard deduction. The largest group used the federal 2026 figure of $16,100 and got $1,137.30. Others used stale or guessed amounts ($13,790–$15,700), added an Arizona personal exemption that does not exist, applied the repealed graduated rates, or made arithmetic or transcription errors."
+us,scenario_018,state_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"Arizona taxes Arizona AGI, here the federal AGI of $61,592.09 (wages minus the traditional 401(k), plus interest, minus the IRA deduction), at a flat 2.5%. The Arizona standard deduction is $16,100 for a single filer in 2026: the $15,750 set by HB 4168, indexed the same way as the federal amount. That beats the $10,637.30 of itemized deductions and leaves taxable income of $45,492.09 and tax of $1,137.30. Most models got the AGI and the 2.5% rate right but used an old or guessed standard deduction ($13,790–$15,750). Others added an exemption a 53-year-old doesn't get, used repealed graduated brackets, took the wrong deduction path, or submitted a number that doesn't match their own work."
us,scenario_018,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_020,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,other,False,,"Texas has no income tax, so the SALT itemized deduction is real estate tax ($13,834) plus the general sales tax deduction from the IRS optional sales tax table. The frozen reference projects that table amount from the IRS's 2023 edition, for $17,090.34 in all, which is under the $40,400 SALT cap and above the $16,100 standard deduction, so it itemizes and the $196 gift falls below the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) uses the IRS 2025 tables instead, whose $1,595 for a single Texas filer at this income leaves SALT below the standard deduction, so the household takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction. The correct path is: deduct the $5,709 alimony above the line to reach AGI of $300,150; deduct the $16,296 of standard and nonitemizer charitable deductions for taxable income of $283,854.19; tax that at 2026 rates, with qualified dividends at 15%; then add $222.64 of NIIT, for $68,334.73, or $68,112.09 without it, the exclusion's corrected value. The frozen reference instead itemizes $17,090.34, for taxable income of $283,059.84 and a total of $68,056.71. The best answers ($68,111.95–$68,404) took the standard deduction, as the convention does; those that also took the $196 nonitemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or of $68,112.09 without it. Many others dropped the alimony deduction, left out NIIT, subtracted ESI premiums, or applied TCJA-sunset rules. Those rules are wrong because OBBBA made the suspension of miscellaneous itemized deductions and personal exemptions permanent. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)."
+us,scenario_020,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,other,False,,"The step that decides this case is whether to itemize. Texas has no state income tax, so the filer can deduct general sales tax from the IRS optional sales tax table, about $3,256. Added to the $13,834 of real estate taxes, that makes $17,090.34 of SALT. It sits well under the 2026 OBBBA cap of $40,400 and beats the $16,100 standard deduction. Once the filer itemizes, the $196 gift falls below the 0.5%-of-AGI floor and the non-itemizer charitable deduction is lost. The other required steps are: deduct the $5,709 of alimony above the line (AGI $300,150), treat the overtime deduction as fully phased out, disallow unreimbursed employee expenses (OBBBA made the suspension permanent, so there is no TCJA sunset), and add the $222.64 NIIT. That gives taxable income of $283,060 and tax of $68,056.71. The strongest models got everything except the sales tax deduction and landed $277–$347 too high. Many others assumed a pre-TCJA sunset, applied a $10,000 SALT cap, dropped the alimony deduction, or subtracted ESI premiums from wages. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)."
us,scenario_020,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_020,payroll_tax,29,llm_error,payroll_tax_base,False,,"The reference is the plain three-component FICA stack on the head's $300,000 of gross wages: 6.2% on the 2026 Social Security wage base of $184,500 ($11,439), 1.45% on all $300,000 ($4,350), and the 0.9% Additional Medicare Tax on the $100,000 above the $200,000 single-filer threshold ($900), with Texas imposing no employee state payroll tax. Two parameter traps separate the reference from the wrong answers: the 2026 wage base is $184,500, not the stale $176,100 (2025), $168,600 (2024), or invented $180,000-$184,200 figures most models substituted; and the payroll base is the full $300,000 of gross wages, so the $8,389 employer-sponsored insurance premium is not shaved off Medicare wages (a $197 understatement) and the $42,857 FLSA overtime premium is already inside the $300,000 rather than added to it. A third cluster derived $16,689 or a near-miss correctly and then submitted a completely different number, discarding their own arithmetic at the reporting step."
+us,scenario_020,payroll_tax,30,llm_error,thresholds_rates,False,,"The correct employee payroll tax has three parts. Social Security is 6.2% of the 2026 wage base of $184,500, or $11,439. Medicare is 1.45% of the full $300,000 in gross wages, or $4,350. Additional Medicare Tax is 0.9% of the $100,000 above the $200,000 single threshold, or $900, and Texas has no employee state payroll tax. Most wrong answers fall into four groups: using an old or guessed wage base such as $176,100, $181,800 or $183,600; subtracting the $8,389 ESI premiums from Medicare wages; adding the $42,857 FLSA overtime premium on top of gross wages that already include overtime; or working out $16,689 correctly and then submitting a different number."
us,scenario_020,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident's state income tax before refundable credits is zero. The nationwide aggregation does not create liability in jurisdictions where the household does not reside."
us,scenario_021,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_021,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Missouri Medicaid eligibility requires qualification through a specific eligibility category; age, disability, immigration status, and modest resources do not independently establish eligibility. The head qualifies through no pathway: MAGI income is 1.72 times FPL, SSI is zero, and the person is not a tax-unit dependent, so Medicaid eligibility is No."
+us,scenario_021,head_medicaid_eligible,6,llm_error,thresholds_rates,False,,"At 85 and on Medicare, the head is not in the MAGI adult expansion group, which covers ages 19–64. The only route left is Missouri's aged/blind/disabled (ABD) pathway, and its income test is 85% of the federal poverty level (FPL). For that test, the full $22,000 of Social Security plus about $5,530 of pension counts as unearned income. After the $20 general disregard, countable income is still about 172% of FPL, roughly double the limit, so medicaid_category comes out as none. Every model treated age and disability, plus assets below the limit, as enough to qualify. None compared the head's actual income with the ABD income limit, and PolicyEngine does not let medical expenses or a spend-down bring countable income down."
us,scenario_021,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to covered wages and self-employment earnings, not Social Security retirement benefits or pension income. With no wages or self-employment income listed, the employee Social Security and Medicare tax base is zero, yielding payroll tax of $0."
us,scenario_021,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_021,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_021,snap,1,llm_error,thresholds_rates,False,,"An elderly or disabled SNAP household is exempt from the gross-income test but must still satisfy the one-person net-income limit. With monthly net income of approximately $2,063 even under the model’s own deductions, the household fails that test, so the calculation stops at ineligibility and produces $0 rather than an allotment."
us,scenario_021,state_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"Missouri tax is computed from Missouri taxable income after applicable deductions and the 2026 taxable-income threshold, not by applying a flat rate directly to taxable pension income. After those steps, this household has no Missouri taxable income and therefore owes $0 before refundable credits."
us,scenario_021,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_022,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is widowed but has no dependent child, so qualifying surviving spouse status is unavailable and the return is filed as single, with the 22% rate starting above $50,400. The prompt lists no state income tax withheld or paid. In the reference the return itemizes $29,735.52: $14,887.85 of SALT, which is $8,829 of property tax plus $6,058.85, the engine's estimate of California income tax withheld (a formula on federal AGI), under the $40,400 cap, and $14,847.67 of charity after the new 0.5%-of-AGI floor. On top of that come the $910 QBI deduction on rental income and the senior deduction phased down to $3,951.05, which leaves $74,552.59 of taxable income. On policyengine-us 2.15.17 with California's 2025 amounts held for 2026, the system that built this reference, the household's California income tax is $2,505.87; the published California reference, $2,439.65, was frozen on policyengine-us 1.755.4 without that hold. With $2,505.87 in SALT the return still itemizes ($26,182.54) and the output is $11,895.23. The household's California output is itself excluded for an engine defect, and with its corrected value, $1,968.81 (recorded on policyengine-us 1.755.4), the output is $12,013.38. With no state income tax paid, SALT takes the engine's general sales tax estimate, $1,155.60 ($963 state plus a $192.60 local estimate for an unlisted locality), and the output is $12,192.29, or $12,234.66 without the local part. The wrong answers split three ways: 16 used joint/QSS brackets, 8 applied the expired pre-TCJA law (personal exemptions, 2% miscellaneous deductions, 15%/25% brackets), and the rest include models that filed single but put no state income tax in SALT or their own California tax estimate ($1,867 to $2,700), or left out the senior deduction or the QBI deduction. Grok 4.7 and Inkling answered $12,192, within $1 of the no-payment value of $12,192.29, though each put its own California tax estimate in SALT. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
+us,scenario_022,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is widowed and has no dependent child, so qualifying-surviving-spouse status does not apply and they file single. As a single filer, they itemize $29,735.52. That total is $8,829 of property tax plus PolicyEngine's ~$6,059 California income tax estimate in SALT, which is well under the OBBBA $40,400 cap, plus $15,393 of charity reduced by the new 0.5%-of-AGI floor. They also get the $910 QBI deduction on rental income and the senior deduction, which phases down 6% above $75,000 MAGI to $3,951.05. That leaves $74,552.59 of taxable income, taxed at 2026 single rates. The wrong answers fall into three main groups: models that used joint surviving-spouse brackets came in too low; single filers that left out the state income tax in SALT, the QBI deduction, or the senior phase-down came in too high; and models that applied pre-TCJA law (personal exemptions, 2%-floor miscellaneous deductions, 15%/25% brackets) were wrong because OBBBA made the TCJA rules permanent. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
us,scenario_022,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_022,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Age 77 establishes an aged status but does not itself confer Medicaid eligibility, and Medicare eligibility or assumed enrollment is not a Medicaid eligibility pathway. The head receives no SSI and qualifies under none of California's Medicaid categories, so the aged status cannot overcome the absence of a qualifying pathway."
us,scenario_022,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_022,payroll_tax,39,llm_error,state_local_rule,False,,"Every model computed the federal parts correctly: 6.2% Social Security ($3,263.37) plus 1.45% Medicare ($763.21), with no Additional Medicare Tax. The whole $684.26 gap is California's mandatory employee State Disability Insurance contribution, which for 2026 is 1.3% of all $52,635 in wages with no wage cap. The models split into two groups. Sixteen left CA SDI out entirely and landed near $4,026.58. The other twenty-two included SDI but used an old or guessed rate: 1.2% ($631.62), 1.1% ($578.99, the 2024 rate) or 0.9% ($473.72, the 2023 rate). One model returned no answer."
+us,scenario_022,payroll_tax,40,llm_error,state_local_rule,False,,"Every model got the federal part right: 6.2% Social Security ($3,263.37) plus 1.45% Medicare ($763.21) on $52,635 of wages, with no Additional Medicare Tax. The missing step is California's mandatory employee State Disability Insurance (SDI) contribution. For 2026 it is 1.3% of all wages, with no wage cap, which is $684.26 and brings the total to $4,710.83. The wrong answers fall into three groups: (1) left out SDI entirely and stopped near $4,026.58; (2) used an old SDI rate, either 2025's 1.2% (giving $4,658.20), 2024's 1.1% (giving $4,605.57) or 2023's 0.9% (giving $4,500.30); or (3) made arithmetic or answer-submission errors."
us,scenario_022,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_022,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"A surviving spouse can use qualifying-surviving-spouse status only with a dependent child in the home, and this person has no dependents. PolicyEngine therefore treats the filer as single: $5,706 standard deduction, single rate brackets, and one personal credit plus one senior exemption credit, totaling $312.93. California AGI is $91,524.41 after subtracting taxable Social Security. Itemized deductions are only charitable gifts ($14,847.67, less than the full $15,393) plus $8,829 of real estate taxes, for $23,676.67; unreimbursed employee business expenses are not deducted. That leaves taxable income of $67,847.75 and gross tax of $2,752.58. The largest group of models used joint/surviving-spouse brackets and double credits, which gives about $650–$1,140. A second group used single brackets but deducted about $6,500 of employee business expenses under a 2% floor, which gives about $1,870–$2,180. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,439.65; the corrected value is 1,968.81. PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (Cal. R&TC 17024.5 (conformity date), 17076)."
+us,scenario_022,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The correct path starts by removing the $17,624.75 of taxable Social Security, leaving CA AGI of $91,524.41. California still allows unreimbursed employee business expenses as a miscellaneous itemized deduction above a 2% floor measured on federal AGI ($8,351 − ~$2,183 = $6,167.56), so itemized deductions are $30,389.97 and taxable income is $61,134.44. The head has no qualifying dependent, so the surviving-spouse flag does not unlock joint/QSS treatment: the single schedule applies, which gives $2,274.81, less a $153 personal credit and a $153 senior exemption credit ($306). Most models went wrong in one of four ways: they used joint/QSS brackets with doubled personal credits, left out the employee-expense deduction or measured its floor against CA AGI, dropped the senior exemption credit, or failed to remove Social Security from CA AGI."
us,scenario_022,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_023,federal_income_tax_before_refundable_credits,41,llm_error,other,False,,"AGI is $22,534 ($14,664 of wages after the traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction). Subtracting the $16,100 OBBBA standard deduction leaves $6,434 of taxable income, which is taxed at 10% for $643.43. No nonrefundable credit applies. For the Saver's Credit, Form 8880 subtracts retirement distributions received during the testing period from contributions, and the $8,000 403(b) distribution wipes out the $3,599 of contributions. The AOTC is zero because no qualified tuition expenses are listed. Most wrong answers of $0 came from applying one of these two credits anyway. The rest used a stale standard deduction (such as $15,000, $15,350, $15,750, or TCJA-sunset amounts with a personal exemption), wrongly excluded employer health premiums from wages, or left out the 403(b) distribution."
+us,scenario_023,federal_income_tax_before_refundable_credits,42,llm_error,other,False,,"AGI is $22,534 ($17,443 wages − $2,778 traditional 401(k) + $8,000 taxable 403(b) distributions − $130 IRA deduction). The OBBBA 2026 single standard deduction of $16,100 leaves $6,434 of taxable income, taxed at 10% for $643. No nonrefundable credit applies. The Saver's Credit is zero because IRC §25B(d)(2) reduces qualified contributions by distributions received in the testing period, and the $8,000 403(b) distribution exceeds the $3,599 of total contributions. The AOTC is zero because no qualified tuition is listed. Most models zeroed out the tax with a phantom Saver's Credit or AOTC. The rest used a stale or pre-OBBBA standard deduction (including the TCJA-expiration personal exemption), wrongly subtracted employer health premiums, or dropped the $8,000 distribution from income."
us,scenario_023,federal_refundable_credits,20,llm_error,credit_phaseout,False,,"This case has two traps, and a model had to avoid both to reach $0. First, the AOTC eligibility flags do not create a credit. The credit is 100% of the first $2,000 of qualified tuition expenses plus 25% of the next $2,000, and no expenses are listed, so the base is $0 and the refundable 40% portion is also $0. Second, the childless EITC phaseout is measured on the greater of earned income or AGI. AGI is about $22,535: wages of $17,443, minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. That is above the 2026 single childless completion point of about $19,540, so the EITC is fully phased out."
us,scenario_023,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"The household qualifies for the FREE meal tier through California’s universal program and categorical eligibility, despite its 1.59 federal-poverty-guideline ratio exceeding the federal income cutoff. The requested output is narrower: it equals 1 only when PolicyEngine returns positive annual free-school-meal support, and this adult-only household has no student receiving meals, so the annual support is zero."
us,scenario_023,head_chip_eligible,2,llm_error,categorical_eligibility,False,,"CHIP eligibility requires being in a covered CHIP category and not already eligible for Medicaid; it is not a general adult coverage pathway based solely on falling below an extended income ceiling. The 28-year-old head qualifies for Medicaid through the WORKING_DISABLED_BUY_IN category, which places them in CHIP category 3 and categorically excludes them from CHIP."
-us,scenario_023,head_medicaid_eligible,24,prompt_ambiguity,categorical_eligibility,False,,"The head is 28, disabled, and working 24 hours a week at $14 an hour. That puts them in California's working-disabled Medicaid buy-in category (medicaid_category = WORKING_DISABLED_BUY_IN), whose income limit is far above the 138% FPL adult-expansion cap. At 1.41x FPL, the head is ineligible under the MAGI adult group but eligible through the buy-in. Every model that gave reasoning tested only the 138% FPL expansion limit and, at most, the SSI-linked aged/blind/disabled pathway. None of them considered the employed-disabled buy-in, even though the facts list both the disability and the employment. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release. Reference depends on an unlisted input; output excluded from scoring (meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require)."
+us,scenario_023,head_medicaid_eligible,25,prompt_ambiguity,categorical_eligibility,False,,"The head is disabled and has a job, so PolicyEngine places them in the working-disabled buy-in category. In California this is the 250% Working Disabled Program, which uses a much higher income ceiling than the 138% FPL adult expansion. The head's income is 1.41x FPL and their only asset is $130 in the bank, so they pass both tests. Nearly every model checked only the 138% FPL adult expansion limit and an SSI-linked aged/blind/disabled pathway, saw income just above 138%, and stopped without checking the working-disabled buy-in. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release. Reference depends on an unlisted input; output excluded from scoring (meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require)."
us,scenario_023,head_medicare_eligible,4,llm_error,age_disability,False,,"The head is 28, so age-based Medicare eligibility at 65 does not apply. For someone under 65, disability-based Medicare requires 24 months of Social Security Disability Insurance (SSDI) entitlement or an end-stage renal disease diagnosis, not just a disability status. This household has no SSDI income and no ESRD, so the head is not eligible. The wrong models treated the `is_disabled` flag by itself as enough for Medicare."
us,scenario_023,head_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_023,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_023,payroll_tax,41,llm_error,thresholds_rates,False,,"The reference has three parts, all applied to the full $17,442.65 of wages: employee Social Security at 6.2% ($1,081.44), Medicare at 1.45% ($252.92), and California SDI at the 2026 employee rate of 1.3% with no wage cap ($226.75). The derivation text says 1%, but $226.75 is exactly 1.3% of the wage base. Most models got FICA right and then missed the SDI piece in one of two ways: they left SDI out, or they used an older rate (0.9% from 2023, 1.1% from 2024, or 1.2% from 2025). A second group shrank the wage base by subtracting the $8,089 employer-sponsored insurance premiums or the $2,778 traditional 401(k) deferral. Neither amount reduces the stated gross wages that serve as the FICA and SDI base."
+us,scenario_023,payroll_tax,42,llm_error,state_local_rule,False,,"The full $17,442.65 of wages is the base for every payroll tax: Social Security at 6.2% ($1,081.44) and Medicare at 1.45% ($252.92), with no Additional Medicare Tax. California's mandatory employee SDI is charged on top of that, with no wage cap, and comes to $226.75, which is 1.3% of wages, the 2026 rate. Traditional 401(k) deferrals and employer-sponsored insurance premiums do not reduce this base, and 403(b) distributions are not wages. Most models got the federal FICA part right. They then either left out CA SDI or used an older or wrong SDI rate (0.9%, 1.0%, 1.1%, 1.13%, 1.2%). Some also subtracted the $8,089 ESI premium or the 401(k) deferral from the wage base."
us,scenario_023,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_023,snap,46,prompt_ambiguity,age_disability,False,,"A ""disabled"" flag alone does not make this a SNAP elderly/disabled household. Under the 7 CFR 271.2 definition, the member must receive a disability-based benefit such as SSI or SSDI, and the head receives neither. Because of that, the household gets no excess medical deduction, and its excess shelter deduction is capped at $744. Monthly gross income is $2,120.25 ($1,453.58 wages plus $666.67 from the 403(b)). Subtracting the $290.72 earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction leaves net income of about $876.51. That gives the frozen reference's $298 − $262.80 = $35.20/month for January–September and $304.68 − $256.50 = $48.18/month for October–December under the engine's projected FY2027 schedule, or $461.34 for the year. The release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $34/month ($408). Most models wrongly treated the head as SNAP-disabled, used an uncapped shelter deduction and a medical deduction, and got large benefits or the maximum allotment. A smaller group applied the 130% gross income test instead of California's 200% FPL broad-based categorical eligibility limit, or left out the shelter deduction, and got $0. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)."
+us,scenario_023,snap,47,prompt_ambiguity,age_disability,False,,"The main trap is the head's 'is disabled' flag. For SNAP, a household counts as elderly/disabled only if a member receives a disability-based benefit such as SSI or SSDI (7 CFR 271.2). Elderly/disabled status is what removes the excess shelter cap and allows the excess medical deduction. The head receives no such benefit, so the shelter deduction stays capped at $744 and no medical deduction applies. Net income is $2,120.25 − $290.72 earned-income deduction − $209 standard deduction − $744 capped shelter = $876.51, which gives $298 − $262.80 = $35.20/month for January–September and $48.18/month for October–December after the FY2027 uprating, or $461.34 for the year. Most models used an uncapped shelter deduction and got near-maximum benefits. The zero-benefit models missed California's broad-based categorical eligibility, which sets the gross income limit at 200% FPL; $2,120/month passes that limit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)."
us,scenario_023,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to satisfy the aged, blind, or disabled categorical gate before income exclusions or benefit-rate calculations apply. The head's unlisted SSI disability status is false under the prompt instructions, so `is_ssi_aged_blind_disabled=False` and SSI is $0 without an income calculation."
-us,scenario_023,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"California AGI is $22,534 ($17,443 wages − $2,778 traditional 401(k) + $8,000 taxable 403(b) − $130 IRA). Subtracting the $5,706 single standard deduction gives $16,828 taxable. Tax at 1% up to $11,079 and 2% above is $225.78. The $153 personal exemption credit and the $60 nonrefundable renter's credit bring it down to exactly $12.78. The answer is a small residual, so it only comes out right with the exact 2025-published parameters and both credits. Models went wrong in four main ways: they declared the tax fully offset even though $225.78 is more than $213 in credits, they dropped the $8,000 403(b) distribution, they left out one of the two credits, or they used 2026-inflated or outdated parameters that moved the residual by a few dollars."
-us,scenario_023,state_refundable_credits,45,llm_error,credit_phaseout,False,,"The filer's only state refundable credit is the childless CalEITC, which comes to $95.15. A 28-year-old filer without children qualifies. The $8,000 in taxable 403(b) distributions is neither earned income nor disqualifying investment income, but it raises AGI to $22,534 (wages $17,443 + $8,000 − $2,778 traditional 401(k) − $130 IRA). In the phase-out range, CalEITC uses whichever is greater, earned income or AGI, so the credit comes from CalEITC's two-segment phase-out schedule at AGI $22,534, not at wages of $17,443. Models went wrong in four ways: they denied eligibility outright, tied CalEITC to the federal EITC, phased the credit out using only wages, or guessed amounts near the maximum credit or above it."
+us,scenario_023,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"California AGI is $22,534.34: wages minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. Subtracting the 2025 single standard deduction of $5,706 leaves $16,828.34 of taxable income. Tax at 1% up to $11,079 and 2% above that is $225.78. The nonrefundable credits ($153 personal exemption credit plus the $60 renter's credit) total only $213, so $12.78 remains. Models missed this small positive amount in four main ways: they claimed the credits wiped out the tax, left out the 403(b) distribution, used projected 2026 or older-year parameters instead of the 2025 values, or dropped or invented a nonrefundable credit."
+us,scenario_023,state_refundable_credits,46,llm_error,credit_phaseout,False,,"California's CalEITC covers childless filers aged 18 and over with earnings up to about $32k in 2026. It does not depend on qualifying for the federal EITC or having children, and it is refundable. In the phase-out range, the credit is computed on the greater of earned income ($17,442.65) or AGI ($22,534.34). This filer's AGI is higher because it includes $8,000 of taxable 403(b) distributions, minus the 401(k) deferral and IRA contribution, so the credit is computed on AGI and comes to $95.15. Models that answered $0 wrongly denied childless or non-federal-EITC eligibility, or thought the credit had already phased out. Models that answered higher amounts ran the phase-out on wages alone, guessed at the credit schedule, or added nonrefundable credits."
us,scenario_023,tanf,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_025,federal_income_tax_before_refundable_credits,30,llm_error,thresholds_rates,False,,"The engine uses the OBBBA-permanent TCJA structure for 2026 MFJ: a $32,200 standard deduction and 10%/12% brackets breaking at $24,800. That gives taxable income of $94,925.30 − $32,200 = $62,725.30 and tax of $2,480 + 12% × $37,925.30 = $7,031.04, with no nonrefundable credits for this childless couple under 65. The wrong answers fall into four groups. Some assumed the TCJA sunset and used pre-TCJA deductions, personal exemptions and a 15% bracket, landing near $8,900–$9,150. Some used the pre-OBBBA 2025 figures, a $30,000 deduction and a $23,850 bracket, and got $7,314. Some used near-miss 2026 parameters. Several computed $7,031 or close to it and then submitted an unrelated number, including $0."
+us,scenario_025,federal_income_tax_before_refundable_credits,31,llm_error,thresholds_rates,False,,"The OBBBA made the TCJA rate structure permanent, so this case only needs three 2026 MFJ parameters: the $32,200 standard deduction, the 10% bracket ending at $24,800, and the 12% rate after that. AGI is $94,925 (spouse wages plus taxable pension). Taxable income is $62,725, so the tax is $2,480 + 12% × $37,925 = $7,031.04, with no nonrefundable credits. Most models missed for one of three reasons: they assumed the TCJA sunset (personal exemptions, a smaller standard deduction and a 15% bracket); they used 2025 or misestimated 2026 thresholds; or they reached about $7,031 and then submitted a different, unsupported final number."
us,scenario_025,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_025,head_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 in the absence of a qualifying disability, ESRD, or other explicitly listed pathway. The 61-year-old head has no such condition, so the age test yields 0 (No)."
-us,scenario_025,payroll_tax,7,llm_error,payroll_tax_base,False,,"The employee-side payroll tax base here is exactly one number: the spouse's $62,725 of gross wages. The head reports no wages at all (only ESI premiums, hours, and medical expenses), the spouse's $32,200 taxable private pension is not FICA wages, employer-sponsored insurance premiums are not netted out of the FICA base in PolicyEngine's employee_payroll_tax, and Ohio imposes no mandatory employee-side state payroll tax (no employee SDI/UI, and state income tax is a separate output). So the answer is 6.2% + 1.45% on $62,725 = $3,888.97 Social Security + $909.52 Medicare = $4,798.48, with no Additional Medicare Tax below the $250,000 MFJ threshold. Every wrong answer either enlarged or shrank that $62,725 base, bolted on a nonexistent state levy or the employer share, or submitted a number that contradicted its own correct arithmetic."
-us,scenario_025,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Ohio AGI is $94,925. Subtract two $1,900 exemptions (the amount for Ohio MAGI above $80,000) to get $91,125 of taxable income. In the 2026 schedule used here, the bracket from $26,050 to $100,000 is taxed at $332 plus 2.75% of the excess over $26,050, which gives $2,121.57. The $200 nonrefundable retirement income credit for the $32,200 pension then brings this to $1,921.57. The largest group of wrong answers (about $1,589.56) left out the $332 base amount. Other models used the wrong exemption amount, subtracted a federal standard deduction or health premiums, used outdated multi-bracket schedules, or claimed a joint filing credit or senior credit the household does not qualify for."
+us,scenario_025,payroll_tax,8,llm_error,payroll_tax_base,False,,"The spouse's $62,725 of wages are the only FICA-covered earnings. They are taxed at 6.2% for Social Security and 1.45% for Medicare, which gives $4,798.48 ($3,888.97 + $909.52). The head has no wages. The $32,200 pension is outside the FICA base. The head's ESI premium is not subtracted from the spouse's wages. Ohio has no employee state payroll tax, and Additional Medicare Tax does not apply. Several models got this computation right but submitted a different number. Others added Ohio income tax, subtracted the ESI premium, used the combined employer-plus-employee rate, or counted wages for the head."
+us,scenario_025,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"The reference has four parts. First, Ohio AGI is federal AGI ($94,925.30) minus the unreimbursed medical care deduction. That deduction is $180.60: $6,500 of premiums plus $800 of other medical expenses, less the 7.5%-of-FAGI floor. The premiums count only above the floor because the head can get employer-paid coverage. Second, two $1,900 exemptions apply because Ohio MAGI is over $80,000. Third, the 2026 rate schedule adds a base amount of about $332 to 2.75% of income over $26,050. Fourth, the $200 retirement income credit applies for the $32,200 pension. The most common wrong answer, $1,589.56, gets the exemptions and the credit right but leaves out the $332 base amount and the medical deduction. Most other answers also use the wrong exemption amount, use the 3%+ brackets from before 2024, subtract a federal standard deduction, or add or drop credits."
us,scenario_026,child1_chip_eligible,5,llm_error,categorical_eligibility,False,,"NC Health Choice merged into NC Medicaid in 2023. North Carolina now covers school-age children up to 211% FPL, plus a 5% disregard, as Medicaid (Medicaid-expansion CHIP) under the OLDER_CHILD category, not as a separate CHIP program. PolicyEngine finds the 11-year-old Medicaid-eligible at this household income. Because CHIP covers only children who fail Medicaid, that Medicaid eligibility makes the child CHIP-ineligible. Every model that answered treated the 211% band as a separate CHIP program sitting above a lower Medicaid limit. Each one called the child CHIP-eligible instead of routing them to Medicaid first and excluding CHIP."
-us,scenario_026,child1_medicaid_eligible,45,llm_error,thresholds_rates,False,,"In April 2023, North Carolina folded its separate CHIP program (NC Health Choice) into Medicaid. Since then, Medicaid for children ages 6–18 covers incomes up to 211% FPL, or 216% with the 5% MAGI disregard, not the old 133% FPL limit. Household MAGI is the spouse's $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA contributions, which gives $82,301, or 2.13x FPL. That is inside the OLDER_CHILD limit, so Child 1 is eligible. Most models used the outdated 133%/138% limit. Others overstated MAGI by double-counting overtime, skipping the pre-tax deductions, or using the 2025 poverty guideline. Others wrongly treated employer-sponsored insurance as a bar to Medicaid, but only CHIP requires a child to be uninsured."
+us,scenario_026,child1_medicaid_eligible,46,llm_error,thresholds_rates,False,,"North Carolina folded NC Health Choice (separate CHIP) into Medicaid. Children ages 6–18 are now covered in the Medicaid older-child category up to 211% FPL, or 216% with the 5-percentage-point MAGI disregard. The household's MAGI is $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA deductions, or $82,301. That is 2.13× the 2026 FPL for five people, which is under the limit, so the 11-year-old qualifies. Most models used the old 133%/138% FPL older-child Medicaid limit. Others overstated income by double-counting overtime or using gross wages, used the wrong poverty guideline, or wrongly treated employer-sponsored insurance as a Medicaid bar. That ESI bar applies only to CHIP, not to Medicaid."
us,scenario_026,child2_chip_eligible,5,llm_error,categorical_eligibility,False,,"Child 2 is 11. In the engine, North Carolina's Medicaid OLDER_CHILD category covers this child at the household's income, so the child is Medicaid-eligible. CHIP only covers children who do not qualify for Medicaid, which makes child2_chip_eligible = No. NC folded NC Health Choice into Medicaid, so the income band that the models called ""CHIP, above Medicaid and up to 211-216% FPL"" is Medicaid coverage for a school-age child. All four models that answered found income below the NC children's limit but assigned the child to CHIP instead of Medicaid, and never applied the rule that Medicaid eligibility excludes CHIP."
-us,scenario_026,child2_medicaid_eligible,45,llm_error,thresholds_rates,False,,"In April 2023, North Carolina folded NC Health Choice (its separate CHIP) into Medicaid. Since then, children ages 6-18 get Medicaid up to 211% FPL, which becomes about 216% after the 5-point MAGI disregard. PolicyEngine puts child2 in the OLDER_CHILD category with household MAGI (spouse wages minus the traditional 401(k) and IRA deductions, about $82,300) at 2.13× the 2026 FPL for a family of five. That is under the older-child limit, so child2 is Medicaid-eligible. Almost every model used the old 133%/138% Medicaid limit (with Health Choice CHIP above it), applied a bare 210-211% cap with no disregard, or misstated MAGI or the FPL. A few also wrongly treated employer coverage as a bar to Medicaid."
+us,scenario_026,child2_medicaid_eligible,46,llm_error,thresholds_rates,False,,"MAGI is the spouse's $85,209 in wages minus the traditional 401(k) and IRA deductions, about $82,300. Against the 2026 FPL for a household of five (about $38,700), that is 2.13 times FPL. NC Health Choice was folded into Medicaid in April 2023, so NC Medicaid now covers children ages 6–18 up to 211% FPL, or 216% after the 5-point MAGI disregard, and 2.13 falls under that limit. The models got this wrong in three main ways: they used the old 133% Medicaid line for ages 6–18 (when 133–211% went to separate CHIP), they compared against 210–211% without the 5-point disregard or measured income against an older, lower FPL, or they treated employer-sponsored insurance as a bar to Medicaid."
us,scenario_026,child3_chip_eligible,7,llm_error,categorical_eligibility,False,,"Every model placed 9-year-old Child 3 in a CHIP band that sits above Medicaid, using the old 133%-to-211% FPL NC Health Choice structure. North Carolina moved NC Health Choice into Medicaid in 2023, and PolicyEngine finds Child 3 Medicaid-eligible under the OLDER_CHILD category. A Medicaid-eligible child is excluded from CHIP by construction, so the answer is No however each model computed MAGI. The decisive step is the Medicaid-first ordering with North Carolina's older-child Medicaid limit, not any CHIP income ceiling."
-us,scenario_026,child3_medicaid_eligible,45,llm_error,thresholds_rates,False,,"NC Health Choice, the state's separate CHIP, was folded into Medicaid on April 1, 2023. Since then, NC covers children ages 6-18 in its Medicaid older-child category up to 211% FPL, or about 216% with the 5% MAGI disregard, instead of the old 133%/138% federal minimum. Household MAGI is $85,209 in wages minus $2,778 in traditional 401(k) and $130 in traditional IRA contributions, which comes to $82,301, or 2.13x the 2026 FPL for five people. That is under the older-child limit, so Child 3 is Medicaid-eligible. Almost every model either used the pre-merger 133%/138% limit, pushed the 138-211% band into CHIP, compared gross wages against 210/211% with no disregard, or wrongly treated employer coverage as blocking Medicaid."
+us,scenario_026,child3_medicaid_eligible,46,llm_error,thresholds_rates,False,,"NC Health Choice, the state's separate CHIP, was folded into Medicaid in April 2023. Since then, NC Medicaid covers children ages 6-18 up to 211% FPL, or about 216% with the 5% MAGI disregard. The household's MAGI is about $82,301: spouse wages of $85,209, which already include overtime, minus the $2,778 traditional 401(k) and $130 traditional IRA deductions. That is 2.13 times the 2026 FPL for five people, so Child 3 qualifies in the OLDER_CHILD category. Most models used the old 133%/138% limit and assumed incomes above it go to a separate CHIP. Others compared gross wages instead of MAGI, left out the disregard, used the wrong FPL, or wrongly treated employer coverage as a bar to Medicaid."
us,scenario_026,federal_income_tax_before_refundable_credits,22,llm_error,taxable_income_or_deductions,False,,"2026 law keeps the TCJA structure — personal exemptions repealed, a $32,200 MFJ standard deduction — and adds the deduction for qualified FLSA overtime premiums, so the $7,746.26 premium comes out of taxable income rather than being stacked onto wages that already include it, leaving $42,354.26 taxable and $4,586.51 of tentative tax. The child tax credit is $2,200 per qualifying child in 2026, and its nonrefundable portion offsets the entire liability first, with the $1,700-per-child ACTC as the residual, so $6,600 of credit for three children drives the answer to exactly $0. The wrong answers split into two clusters: models that applied the expired pre-TCJA regime (a reverted ~$16,600 standard deduction, ~$5,200 personal exemptions, a 15% bracket, a $1,000-per-child credit), and models that got the deduction stack roughly right but capped the nonrefundable CTC at $500–$1,100 per child by treating the refundable ACTC as consumed first. Every wrong model ignored the qualified-overtime deduction, and three double-counted the $7,746 premium on top of the $85,209 gross wage figure."
-us,scenario_026,federal_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Under 2026 OBBBA law the CTC is $2,200 per child ($6,600 for three), refundable up to $1,700 per child. AGI is $82,301: $85,209 in wages minus the $2,778 traditional 401(k) and the $130 traditional IRA. Subtracting the $32,200 MFJ standard deduction and the $7,746 qualified overtime (FLSA premium) deduction leaves taxable income of about $42,355. At the 10% and 12% rates, pre-credit tax is about $4,586.51, so $2,013.49 of the CTC goes unused. That amount is under both the $5,100 per-child cap and the 15%-of-earnings limit, so all of it is refundable ACTC, and EITC is zero at this income. The wrong answers came from four errors: leaving out the overtime deduction (which pushes tax toward $5,500), using pre-OBBBA CTC amounts ($1,000 or $2,000 per child), saying the tax absorbs the whole credit when tax is below the credit, and adding deductions or credits that do not apply (auto-loan interest, ESI premiums, Saver's Credit)."
+us,scenario_026,federal_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"AGI is $82,301 ($85,209 wages minus the $2,778 traditional 401(k) and $130 traditional IRA). Taxable income is about $42,355 after the $32,200 2026 MFJ standard deduction and the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. That makes pre-credit tax about $4,586.51. Under OBBBA, the CTC is $2,200 per child, or $6,600, so $2,013.49 of credit is left unused after offsetting tax. That amount is refundable because it falls below the $1,700-per-child cap and the 15%-of-earnings phase-in limit, and EITC is $0. Most models missed the overtime deduction, used the outdated $2,000 or sunset $1,000 per-child CTC, or wrongly claimed that the tax fully absorbed the credit."
us,scenario_026,free_school_meals_eligible,1,llm_error,thresholds_rates,False,,"Free school meals require income within the free-meal limit or categorical eligibility through a qualifying program. This household's countable income of $85,208.82 equals 220% of the $38,680 federal poverty guideline, exceeds even the 185% reduced-price limit, and has no categorical eligibility, so its school-meal tier is paid."
us,scenario_026,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_026,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_026,payroll_tax,10,llm_error,payroll_tax_base,False,,"The entire liability is 6.2% plus 1.45% on the Spouse's $85,209 of gross wages: $5,282.95 + $1,235.53 = $6,518.47, with no Additional Medicare Tax below the $250,000 MFJ threshold and no mandatory North Carolina employee payroll tax. The trap is the base: the separately listed $7,746 FSLA overtime premium is a decomposition of those same wages, not extra earnings, and the $21,208 employer-sponsored insurance premium sits on the Head, who has zero wages, while employee FICA is levied on gross employment income with no Section 125 exclusion. Five models moved the base — inflating it with the overtime premium, shrinking it by the ESI premium, or both — one invented a state payroll tax, one included the employer Social Security share, one zeroed the base outright, and one lost the value to a $1.48 arithmetic slip."
+us,scenario_026,payroll_tax,11,llm_error,payroll_tax_base,False,,"The only earner, the spouse, has $85,209 in gross wages, and the prompt says that amount already includes overtime. Employee FICA is therefore 6.2% Social Security ($5,282.95) plus 1.45% Medicare ($1,235.53), for $6,518.47. The 2026 Social Security wage cap, the Additional Medicare Tax threshold and any NC employee payroll tax don't apply, since the wages are below both thresholds and NC has no such tax. Most wrong answers added the $7,746 FLSA overtime premium on top of the gross wages, subtracted the head's $21,208 employer insurance premiums from the FICA base, counted NC income tax as payroll tax, or got the arithmetic wrong."
us,scenario_026,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price school meals require household income at or below 185% of the federal poverty guideline or a qualifying categorical-eligibility pathway. The household's school-meal income is $85,209, equal to 2.20 times the guideline, and it has neither universal-free-meal coverage nor categorical eligibility, so its tier is PAID."
us,scenario_026,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_026,state_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI. Here that is $82,300.52: the spouse's wages, which already include the overtime premium, minus the traditional 401(k) and IRA contributions. From that, subtract the $25,500 joint standard deduction and the NC child deduction. For joint filers, the child deduction is tiered by AGI, and the $80k–$100k tier gives $1,500 per child, or $4,500 for three children. The resulting $52,300.52 is taxed at the flat 3.99% rate for 2026. Most models got this wrong in one of four ways: they left out the child deduction, used the wrong AGI tier ($500, $1,000, $2,000, $2,500, $3,000, or $4,000 per child), added the FLSA overtime premium on top of wages or subtracted it from them, or used an outdated rate or standard deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI of $82,300.52. That is the spouse's $85,209 in wages, which already include the overtime premium, minus the traditional 401(k) and IRA contributions. NC then subtracts two deductions: the $25,500 MFJ standard deduction and the income-tiered NC child deduction. For MFJ filers with AGI between $80,001 and $100,000 the child deduction is $1,500 per child, or $4,500 for three children, which leaves $52,300.52 taxed at the flat 3.99% rate. Most models left out the child deduction or used the wrong income tier. Others got AGI wrong: some counted the FLSA overtime twice, some subtracted it, some subtracted the head's ESI premiums, and some ignored the retirement contributions. A few used the wrong rate."
us,scenario_026,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The household has no positive federal income tax before refundable credits because taxable income and every additional component of that output are zero. A missing submission is a parse-contract failure, while a negative submission contradicts both the model's own zero-tax derivation and the output's nonnegative liability construction."
us,scenario_027,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -214,10 +216,10 @@ us,scenario_027,local_income_tax,1,parse_contract_failure,missing_output,False,,
us,scenario_027,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_027,snap,45,llm_error,categorical_eligibility,False,,"Connecticut's broad-based categorical eligibility (TANF non-cash pathway) makes this household categorically eligible. Gross income is 1.44× the poverty guideline, below CT's 200% limit, so the asset test and the 100% FPL net income test do not block eligibility. Under 7 CFR 273.10(e)(2)(ii)(C), categorically eligible households of one or two people receive the minimum allotment even when the formula gives zero. Here 30% of the $2,118 net income is $636, which exceeds the $546 maximum, so the FY2026 minimum of $24/month applies, or $288 a year. Most models ended at $0: they either applied the $4,250–$4,500 elderly/disabled asset limit or the net income test, or they stopped when the formula benefit came out to zero. A few applied the minimum but used the older $23 amount."
+us,scenario_027,snap,46,llm_error,categorical_eligibility,False,,"Connecticut uses broad-based categorical eligibility (BBCE) through TANF non-cash benefits. Gross income of $2,536.25/month is about 1.44× poverty, below the BBCE limit, so the household is categorically eligible. That removes the federal asset test (the $48,000 doesn't count against it) and the federal 100% FPL net income test. Under the regular formula, the expected contribution is $636 (30% of $2,118 net income), which exceeds the $546 two-person maximum, so the formula benefit is $0. But eligible one- and two-person households get the minimum allotment: 8% of the FY2026 one-person maximum of $298, which is $24/month, or $288/year. Most models applied the $4,250–$4,500 elderly/disabled asset limit or the net income test and returned $0. Others recognized categorical eligibility but left out the minimum allotment or used FY2025's $23 minimum."
us,scenario_027,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,spouse_medicaid_eligible,13,llm_error,categorical_eligibility,False,,"The spouse, 39, qualifies through no Medicaid category in the engine. For the adult expansion group, Medicaid MAGI adds back all non-taxable Social Security benefits, so the head's $30,160 SSDI plus $275 interest gives household MAGI of $30,435. That is 1.41x FPL, above the 138% limit. Being blind and disabled confers no separate category either: the spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits. Models said yes by either dropping Social Security from MAGI or treating blindness and disability as enough on their own under an aged, blind and disabled (ABD, HUSKY C) pathway."
-us,scenario_027,spouse_medicare_eligible,11,llm_error,age_disability,False,,"Under 65, the only Medicare pathways are 24 months of Social Security disability benefit receipt, ESRD, or ALS. A disability or blindness flag does not qualify a person on its own. The 39-year-old spouse has no Social Security disability income of their own; the $30,160 SSDI belongs to the head. The spouse also cannot borrow the head's SSDI entitlement, because Medicare disability coverage requires the person's own entitlement to disability benefits. PolicyEngine therefore marks only the 68-year-old head as Medicare eligible. Every model that answered treated the spouse's 'is disabled' flag as enough for Medicare."
+us,scenario_027,spouse_medicare_eligible,12,llm_error,age_disability,False,,"Medicare eligibility requires age 65+, ESRD/ALS, or 24 months of the person's own Social Security disability benefit receipt. A disability flag alone does not qualify anyone. The 39-year-old spouse is flagged blind and disabled but has $0 Social Security disability income of their own; the $30,160 SSDI belongs to the head. The spouse therefore has no Medicare pathway. The models mistook the disability flag for SSDI entitlement, and some credited the head's SSDI to the spouse."
us,scenario_027,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -241,28 +243,28 @@ us,scenario_028,child3_head_start_eligible,1,parse_contract_failure,missing_outp
us,scenario_028,child3_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid and CHIP are separate eligibility outputs, so CHIP coverage at higher income levels does not establish Medicaid eligibility. At 1.82 times FPL, child3 exceeds every applicable Medicaid income threshold and qualifies under no Medicaid category; employer-sponsored insurance and zero SSI provide no alternative Medicaid pathway."
us,scenario_028,child3_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old fails the categorical age requirement. The household also has approximately $61,277 of income, exceeding the applicable 185%-of-poverty WIC income limit; Child 3 therefore fails both the categorical and income tests."
-us,scenario_028,federal_income_tax_before_refundable_credits,19,llm_error,credit_phaseout,False,,"The decisive step is the ordering and size of the 2026 child tax credit: AGI of $60,010 (wages $60,000 plus $10 of interest; child support received is excluded from gross income) less the $24,150 head-of-household standard deduction leaves $35,860 of taxable income, taxed at 10% on the first $17,700 and 12% on the remaining $18,160 for $3,949.20. Three qualifying children generate $6,600 of CTC at the 2026 rate of $2,200 each, and the nonrefundable CTC is applied first against the full liability, zeroing it out; the refundable ACTC is the $2,650.80 residual, bounded by the $1,700-per-child cap and 15% of earnings above $2,500 ($8,625), neither of which binds. Every wrong answer either priced 2026 with lapsed pre-TCJA parameters (personal exemptions, a ~$12,000 standard deduction, a $1,000-per-child CTC) or reversed the ordering by carving a fixed refundable ACTC out of the credit first and applying only the stub nonrefundably."
-us,scenario_028,federal_refundable_credits,41,llm_error,credit_phaseout,False,,"The reference has two parts. First, OBBBA sets the 2026 CTC at $2,200 per child, so the credit is $6,600. Head-of-household tax on $60,010 AGI is $3,949.20 ($24,150 standard deduction, $35,860 taxable income, 10% bracket to $17,700, then 12%). The refundable ACTC is the unused $2,650.80, which is below both the $5,100 cap ($1,700 per child) and the $8,625 earned-income limit. Second, the 3-child EITC phases down from $8,231 at 21.06% above $23,890 and leaves $624.13 at $60,010, because it does not fully phase out until about $62,974. The wrong answers split into three groups: models that zeroed the EITC by placing the phase-out end below $60,010; models that used the pre-OBBBA $2,000 per child (or $1,000) CTC, or misstated the pre-credit tax; and models that paid the full $5,100 refundable cap without limiting the ACTC to the CTC left after offsetting tax."
+us,scenario_028,federal_income_tax_before_refundable_credits,20,llm_error,credit_phaseout,False,,"Under 2026 law (OBBBA made the TCJA structure permanent), the HoH filer has AGI of $60,010. Child support is not taxable. The $24,150 standard deduction leaves $35,860 of taxable income and $3,949.20 of tax. The full $2,200-per-child CTC ($6,600) is applied first as a nonrefundable credit up to that liability, which zeroes it out. Only the leftover $2,650.80 is refundable ACTC. The wrong answers come from three mistakes: assuming TCJA expired (personal exemptions and a $1,000 CTC), carving the refundable ACTC out of the CTC before applying the nonrefundable portion, or applying only part of the CTC against the liability."
+us,scenario_028,federal_refundable_credits,42,llm_error,credit_phaseout,False,,"The reference has two parts. The first is the refundable CTC: OBBBA's 2026 CTC is $2,200 per child, so $6,600 for three children. AGI is $60,010; after the $24,150 HoH standard deduction, taxable income is $35,860 and pre-credit tax is $3,949.20. The unused $2,650.80 is refundable because it is below the $5,100 cap ($1,700 per child) and below the 15% earned-income limit. The second part is the EITC, which has not fully phased out at $60,010: $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. Most wrong answers fall into one of a few errors: using the pre-OBBBA $2,000 or post-TCJA $1,000 per-child credit, paying the full $1,700/child cap without limiting it to unused CTC, saying the CTC was fully absorbed by tax, dropping the EITC by assuming the 3-child phaseout ends below $60,010, or using the wrong 2026 EITC maximum or phaseout threshold."
us,scenario_028,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_028,payroll_tax,36,llm_error,state_local_rule,False,,"Employee FICA on $60,000 of wages is $3,720 in Social Security plus $870 in Medicare, or $4,590, with no Additional Medicare Tax. Pennsylvania is one of the few states that withholds an employee unemployment compensation contribution: 0.07% of total gross wages with no wage cap, which is $42 here. That brings the total to $4,632. Almost every model stopped at FICA and either ignored the PA employee UC withholding or said outright that PA has no employee payroll tax. One model applied the UC rate to the wrong base, one made an arithmetic slip, one derived $4,632 but submitted a different number, and two returned no answer."
+us,scenario_028,payroll_tax,37,llm_error,state_local_rule,False,,"Pennsylvania is one of the few states that requires employees to pay unemployment compensation tax. The employee rate is 0.07% of all wages with no wage cap, which comes to $42 on $60,000. Adding that to FICA ($3,720 Social Security + $870 Medicare) gives $4,632. Almost every model stopped at $4,590 because it assumed PA has no mandatory employee payroll tax. A few others made arithmetic or base errors, or gave no answer."
us,scenario_028,reduced_price_school_meals_eligible,8,llm_error,thresholds_rates,False,,"School-meal countable income is $61,277 because child support received counts alongside wages and interest. Against the $33,000 poverty guideline for a household of four, that is 1.86 times the guideline, $227 above the $61,050 reduced-price cutoff (185%), so the household is in the paid tier. Models that dropped the $1,267 of child support used $60,010, which is 1.82 times $33,000 (the same figure as the trace's school_meal_fpg_ratio), and landed under the line. Models that did count $61,277 overstated the 185% threshold, putting it somewhere between about $61,300 and $61,975 instead of $61,050."
us,scenario_028,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_028,state_income_tax_before_refundable_credits,10,llm_error,taxable_income_or_deductions,False,,"Pennsylvania's personal income tax is a flat 3.07% on the eight enumerated classes of income with no standard deduction, no personal exemptions, and no filing-status allowances; the base here is $60,000 of compensation plus $10 of taxable interest, and the $1,267 of child support is not a taxable class, giving 3.07% × $60,010 = $1,842.31. The only income-based relief, the Schedule SP tax forgiveness credit, is unavailable because eligibility income of $61,277 far exceeds the $35,000 limit for an unmarried claimant with three dependents ($6,500 + 3 × $9,500). Most wrong models imported federal deduction or exemption concepts into PA's no-deduction structure or fabricated a base or rate; the remainder either failed to execute the multiplication or subtracted a credit that is refundable and zero-valued here."
-us,scenario_028,state_refundable_credits,43,llm_error,state_local_rule,False,,"Starting in tax year 2025, Pennsylvania has a refundable state EITC, the Working Pennsylvanians Tax Credit, equal to 10% of the federal EITC. This head of household has three qualifying children, $60,000 of earned income and $60,010 of AGI (child support is excluded). That puts the household near the end of the federal EITC phase-out, so the federal credit is $624.13 and the PA refundable credit is $62.41. Almost every model relied on the pre-2025 rule that PA has no refundable income tax credit. They looked only at Tax Forgiveness, the CDCC or the Property Tax/Rent Rebate and answered $0. The one model that used the 10% PA EITC overestimated the federal EITC."
+us,scenario_028,state_income_tax_before_refundable_credits,11,llm_error,taxable_income_or_deductions,False,,"Pennsylvania's personal income tax is a flat 3.07% on the gross amount of each taxable income class. PA has no standard deduction and no personal or dependent exemptions, and child support is not taxable. The correct tax is 3.07% × ($60,000 wages + $10 interest) = $1,842.31. The household gets no tax forgiveness: eligibility income of about $61,277 (child support included) is far above the $35,000 full-forgiveness limit for an unmarried claimant with three dependents ($6,500 + 3 × $9,500), and partial forgiveness ends about $2,250 above that limit. The wrong answers came from invented deductions or exemptions, invented credits or forgiveness, arithmetic slips, or a submitted value that did not match the model's own correct reasoning."
+us,scenario_028,state_refundable_credits,44,llm_error,state_local_rule,False,,"Pennsylvania's FY2025-26 budget created the Working Pennsylvanians Tax Credit. It is a refundable state EITC equal to 10% of the federal EITC, effective from tax year 2025. This head of household has three children and about $60,010 of AGI, which sits near the end of the 2026 federal EITC phase-out, so the federal EITC is $624.13 and the PA refundable credit is $62.41. Nearly every model relied on the outdated rule that PA has no refundable credits, or mistook Tax Forgiveness or the child and dependent care credit for the only relevant programs, and answered $0. One model applied the 10% rule correctly but used an imprecise federal EITC."
us,scenario_028,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_029,federal_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only AGI item in this household is $312 of taxable interest: ""financial assistance"" is a non-taxable assistance receipt in PolicyEngine's income taxonomy and feeds no AGI component, so the $50,000 never reaches the 1040. With AGI of $312 against a 2026 single standard deduction of roughly $16,100 under the permanent OBBBA rules, taxable income is $0 and federal income tax before refundable credits is $0. All three models converted the $50,000 assistance figure into taxable income (wages or other ordinary income) and then ran a full bracket computation on roughly $50,312 of AGI, which is the single step that separates every wrong answer from the reference."
-us,scenario_029,head_medicaid_eligible,19,llm_error,taxable_income_or_deductions,False,,"The case turns on what enters Medicaid MAGI: the $50,000 of financial assistance is a non-taxable receipt excluded from MAGI, so the head's countable income is the $312 of taxable interest alone — 0.02 x FPL — and Oklahoma, which adopted ACA adult expansion under State Question 802 effective July 1, 2021, covers childless adults 19-64 up to 138% FPL. Fifteen of the nineteen models folded the $50,000 into countable income to manufacture ~$50,312 and compared that against roughly $21,000-$21,600. Two more (gpt-5.4-nano, qwen3.8-max) applied pre-ACA categorical framing, demanding pregnancy, disability, or a dependent child when the expansion group itself is the pathway. Secondary traps drew several models in as well: asset tests on the $54,985 of bank and stock holdings, which 42 CFR 435.603(g) bars for MAGI groups, and employer-sponsored insurance, which disqualifies from the premium tax credit rather than from Medicaid."
+us,scenario_029,head_medicaid_eligible,20,llm_error,taxable_income_or_deductions,False,,"Oklahoma expanded Medicaid in 2021, so a 21-year-old non-dependent adult qualifies in the ACA adult group if MAGI is at or below 138% FPL. The $50,000 of financial assistance is a non-taxable transfer. It is not part of AGI or MAGI, so MAGI is only the $312 of taxable interest, about 0.02x FPL. MAGI categories have no asset test, and having employer-sponsored insurance does not block Medicaid eligibility. Most models wrongly added the $50,000 to income. The rest wrongly said Oklahoma never expanded Medicaid, required a disability, pregnancy or parent pathway, or applied asset limits."
us,scenario_029,payroll_tax,1,llm_error,payroll_tax_base,False,,"The household lists no wages at all: its only income items are $312 of taxable interest and $50,000 of financial assistance, alongside assets and expenses. Employee Social Security and Medicare taxes attach exclusively to wages and salaries under FICA (IRC §3101, on ""wages"" as defined in §3121(a)); interest is unearned investment income and financial assistance is a non-employment transfer, and neither enters the FICA wage base. With a zero wage base, employee OASDI, employee HI, and Additional Medicare Tax are each $0, giving a household employee-side payroll tax of $0."
us,scenario_029,snap,1,llm_error,asset_resource,False,,"SNAP eligibility requires satisfying Oklahoma’s applicable resource test before the benefit formula is applied. This one-person household has $4,250 in bank assets and $50,735 in stock assets, so it fails the resource limit and receives no SNAP benefit regardless of its low interest income or the maximum allotment."
us,scenario_029,state_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only taxable income in this household is $312 of taxable interest; the $50,000 of ""financial assistance"" is a non-taxable transfer that never enters federal AGI, and Oklahoma's income tax starts from federal AGI, while the $4,250 bank balance and $50,735 of stock are assets that generate no income under the ""treat any unlisted numeric input as 0"" instruction. Against Oklahoma's single standard deduction plus the $1,000 personal exemption, $312 of interest leaves zero Oklahoma taxable income, so ok_income_tax_before_refundable_credits — and therefore the 51-jurisdiction sum — is $0. Every wrong answer is produced by the same step: adding the $50,000 financial assistance into Oklahoma AGI and then applying the graduated OK rate schedule to roughly $37,000–$43,000 of invented taxable income."
-us,scenario_029,state_refundable_credits,44,llm_error,state_local_rule,False,,"The $40 is Oklahoma's refundable Sales Tax Relief Credit (ok_stc): $40 per personal exemption for a household with gross income under the $20,000 limit, filed by an adult head. The household's gross income for the credit is only the $312 of taxable interest. The $50,000 of financial assistance is not part of that income measure, and the credit has no earned-income test, no asset test, and no requirement to have dependents or to be 65+ or disabled. Every model answered $0. Some ignored the Sales Tax Relief Credit and looked only at the EITC. Others counted the $50,000 of financial assistance (or the assets) toward the $20,000 limit. A few invented an age, disability, or dependent requirement."
+us,scenario_029,state_refundable_credits,45,llm_error,categorical_eligibility,False,,"Oklahoma's refundable Sales Tax Relief Credit (ok_stc) pays $40 per personal exemption to any adult filer whose gross household income is at or below $20,000. It does not depend on earned income, dependents, age 65+, disability, or tax liability; those factors only raise the limit to $50,000. PolicyEngine counts gross income from the household's income sources, which here is only the $312 of taxable interest. The $50,000 of financial assistance is not counted, so this single 21-year-old filer with one exemption receives $40. Most models looked only at the EITC, saw no earned income, and stopped. A smaller group named the credit but added the $50,000 of financial assistance to gross income, which pushed the filer over the $20,000 limit."
us,scenario_030,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Only the $13,000 of wages enters adjusted gross income in this household. The listed educational assistance and financial assistance do not add $17,000 of taxable income, so AGI remains below the 2026 single-filer standard deduction and taxable income and pre-refundable-credit federal income tax are both zero."
us,scenario_030,federal_refundable_credits,12,llm_error,categorical_eligibility,False,,"A childless EITC claimant in 2026 must satisfy the minimum-age requirement, and this 23-year-old does not. Pregnancy does not create a qualifying child for EITC or CTC purposes, so every evaluated refundable credit component is zero."
us,scenario_030,head_chip_eligible,5,llm_error,categorical_eligibility,False,,The head qualifies for Medicaid under the pregnant-person category. Medicaid eligibility precludes CHIP eligibility under PolicyEngine because CHIP covers qualifying children who do not qualify for Medicaid; the Texas CHIP perinatal income threshold therefore never determines this output.
@@ -271,27 +273,27 @@ us,scenario_030,head_wic_eligible,2,llm_error,household_unit_or_filing_status,Fa
us,scenario_030,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_030,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on the $13,000 wage base. Those components are $806.00 and $188.50, totaling $994.50 without whole-dollar rounding."
us,scenario_030,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_030,snap,46,llm_error,categorical_eligibility,False,,"The SNAP unit has one person. The $12,000 of financial assistance counts as unearned income, so gross income is $2,083.33 a month, about 160% of the $1,304.17 poverty guideline. Texas has broad-based categorical eligibility through TANF non-cash benefits, with a 165% FPL gross limit (about $2,152 a month), so this household is categorically eligible and does not face the 130% gross test or the 100% net test. Net income is $1,658, so the expected contribution ($498) is more than the $298 maximum allotment. Because the household is an eligible 1-person unit, it still gets the $24 minimum allotment ($288 a year). Most models either used the 130% test or net test and got $0, or left out the financial assistance and got a formula benefit of about $100 a month. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))."
+us,scenario_030,snap,47,llm_error,categorical_eligibility,False,,"The SNAP unit is one person. Countable income is $1,083.33 a month in wages plus $1,000 a month in financial assistance, counted as unearned income. The $5,000 of educational assistance is excluded. Gross income is $2,083.33 a month. That is above the regular 130% FPL limit but below Texas's broad-based categorical eligibility limit of 165% of the FY2026 guideline (165% × $1,304.17 = $2,151.88). Because the household is categorically eligible through TANF non-cash eligibility, the gross and net income tests are waived. The expected contribution ($498 a month) exceeds the $298 maximum allotment, so this one-person household gets the $24 monthly minimum allotment, or $288 a year. Most models either stopped at the 130% gross limit or the net income test, or skipped the minimum allotment. A smaller group left out the $12,000 of financial assistance and calculated a large regular benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))."
us,scenario_030,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_030,tanf,2,llm_error,categorical_eligibility,False,,Texas TANF requires a family with a dependent child; pregnancy alone does not turn the unborn child into a qualifying dependent child or create a two-person assistance unit for this annual TANF calculation. The single adult therefore has no qualifying child and receives zero TANF.
us,scenario_031,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The $1,165 alimony expense reduces $14,408 of IRA and pension income to $13,243 of adjusted gross income, while Social Security remains nontaxable. The applicable single-filer standard deduction, including the age-65 addition, exceeds that income, producing zero taxable income and therefore zero federal income tax before refundable credits."
-us,scenario_031,head_medicaid_eligible,43,llm_error,taxable_income_or_deductions,False,,"At 67 the head is aged (65 or older), which alone meets the category condition of the engine's optional senior-or-disabled pathway; it needs no disability or SSI receipt. The adult expansion group (ages 19 to 64, no Medicare eligibility) does not apply, but in the engine's Medicaid rules Medicare eligibility bars only that group. The pathway counts income under SSI rules: Social Security ($9,445.47, in full), the $13,608 pension and the $800 IRA distribution give $23,853.47; the $1,165 of alimony paid is not subtracted. In place of SSI's $20 monthly general income exclusion, the engine applies California's $230 monthly disregard ($2,760 a year), leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline of $15,960). The $4,200 in bank assets is far below California's $130,000 asset limit for the pathway. The engine also models a $2,434.80 Medicare Part B premium for the head, but countable income does not subtract it: with the premium at zero, or the head not enrolled in Medicare, countable income is still $21,093.47. Most models (29 of 43) miscounted income: most of those compared about $23,853 with the limit, a few applied only SSI's $20 exclusion or an exclusion of unstated size, and several asserted without figures that income was too high. The rest used the wrong threshold (100% of the poverty guideline or an SSI income limit), applied SSI's $2,000 resource limit or otherwise judged the $4,200 too much, or denied the head a category, by treating Medicare eligibility or the lack of SSI receipt or a disability as disqualifying, or by testing only the adult expansion group."
+us,scenario_031,head_medicaid_eligible,44,llm_error,state_local_rule,False,,"At 67, the head qualifies by age alone for California's Aged & Disabled pathway, the optional senior-or-disabled category; neither SSI receipt nor a disability is needed, and Medicare eligibility bars only the 19–64 expansion group. Income is counted under SSI rules: $23,853.47 gross, with alimony paid not subtracted. California then subtracts a $230/month ($2,760/year) disregard in place of SSI's $20 general exclusion. That leaves $21,093.47, which is under 138% of the 2026 FPL ($22,024.80). Resources of $4,200 are far below California's $130,000 individual limit. Almost every model compared gross income, or income less only the $20/month exclusion, against 138% FPL. The rest denied the aged category, treated Medicare as a bar, or used the $2,000 SSI resource limit."
us,scenario_031,snap,6,llm_error,categorical_eligibility,False,,"The head is 67, so this is an elderly household. PolicyEngine does not let California's 200% FPL broad-based categorical eligibility skip the net income test for this household. Monthly gross income is $1,987.75 (Social Security, pension and IRA). The only deduction is the $209 standard deduction. There are no shelter costs, and the $50/yr OTC medical expense is below the $35/mo threshold, so net income is about $1,779/mo. That is well above the 1-person net limit of about $1,305/mo (100% FPL), so the household is not SNAP-eligible. The one/two-person minimum benefit only applies to eligible households, so SNAP is $0. Five models treated BBCE as automatic eligibility and paid the minimum benefit. The sixth undercounted income."
us,scenario_031,state_income_tax_before_refundable_credits,1,llm_error,credit_phaseout,False,,"California first computes $75.37 of tax on $7,537 of taxable income, then applies $312.93 of nonrefundable credits, including the aged-or-blind exemption credit. Those credits fully eliminate the liability, so state income tax after nonrefundable credits and before refundable credits is $0."
us,scenario_032,child1_chip_eligible,13,llm_error,health_coverage,False,,"CHIP is residual by statute: 42 U.S.C. 1397jj(b)(1)(A) and 42 CFR 457.310 exclude from ""targeted low-income child"" any child who is eligible for Medicaid, so the CHIP question is settled by first running the Medicaid test. Minnesota's Medical Assistance covers children ages 6-18 — PolicyEngine's OLDER_CHILD category — far above this household's $42,664 for three people (~160% FPL), so the child is Medicaid-eligible and thereby barred from CHIP. Every wrong model instead applied a standalone CHIP income ceiling, most of them quoting Minnesota's 275%/283% FPL child limits, which are the Medical Assistance and MinnesotaCare bands rather than CHIP bands. They read ""income below the child-coverage limit"" as qualifying when it is precisely what disqualifies: several stated the Medicaid fact outright and then used it backwards."
us,scenario_032,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_032,child1_head_start_eligible,7,llm_error,categorical_eligibility,False,,"Head Start eligibility requires an applicable categorical pathway or income at or below the tax unit's federal poverty guideline. The household has no categorical pathway, and its $30,916 adjusted gross income exceeds the $27,320 guideline, so the child is not eligible."
+us,scenario_032,child1_head_start_eligible,8,llm_error,thresholds_rates,False,,"Head Start income eligibility compares household income with 100% of the federal poverty guideline. Here AGI is $30,916: $29,000 in wages plus $1,916 of taxable Social Security dependent benefits. The guideline for a family of three is $27,320, so the household is over the limit. The household also has no categorical pathway: no SSI, no TANF, no foster care and no homelessness. The models that said Yes either looked only at the child's age, guessed that the income was 'low enough' without comparing it to the guideline, used a 130–150% FPL limit that Head Start doesn't use, or treated the program-level allowance for over-income enrollment as if it made this child eligible."
us,scenario_032,child1_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"Minnesota applies an age-specific Medicaid MAGI pathway to the six-year-old, whom the engine classifies in the OLDER_CHILD category. At 1.56 times FPL, the child satisfies that category's Medicaid income limit; the adult expansion threshold of 138% FPL does not control child eligibility, and CHIP is not substituted for Medicaid when the Medicaid test is met."
us,scenario_032,child1_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility is determined separately for each person and does not pass from a Social Security beneficiary to a dependent. The six-year-old child is under age 65, and the stated facts include no end-stage renal disease, ALS, disability-based Medicare entitlement, or other qualifying pathway, so the child's eligibility is No."
us,scenario_032,child1_wic_eligible,6,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday; a six-year-old is outside the federally eligible categorical group. Income eligibility cannot override this age restriction, so the child is ineligible regardless of whether household income falls below 185% of the federal poverty guidelines."
us,scenario_032,federal_income_tax_before_refundable_credits,3,llm_error,thresholds_rates,False,,"The decisive rule is the 2026 federal parameter regime: the household receives a $2,200 CTC, split into $500 nonrefundable and $1,700 refundable. The $500 nonrefundable portion fully eliminates pre-credit income tax, while the refundable CTC and $4,427 EITC are excluded from federal_income_tax_before_refundable_credits."
-us,scenario_032,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household files MFJ with one qualifying child. Earned income is $29,000. AGI is $30,916: wages plus $1,916 of taxable Social Security, because provisional income of $35,832 exceeds the $32,000 base but stays below $44,000. Both figures sit below the 2026 MFJ one-child EITC phase-out start (above $31,000), so the EITC is the full 2026 maximum of $4,427. Tax before credits is $0 because AGI is below the $32,200 standard deduction, and 15% of ($29,000 - $2,500) = $3,975 exceeds the cap. The refundable CTC is therefore the full OBBBA refundable cap of $1,700 out of the $2,200 credit, for a total of $6,127. The wrong answers split into three groups: models that used the expired post-TCJA-sunset $1,000 ACTC, models that phased out the EITC using a single-filer or stale threshold, and models that misstated the EITC maximum or the $1,700 refundable cap."
+us,scenario_032,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"For 2026, the household gets the full one-child EITC maximum of $4,427. Earned income is $29,000, and AGI is $30,916 after adding $1,916 of taxable Social Security (50% of provisional income above $32,000). AGI stays below the married-filing-jointly phaseout start of about $31,160. Under OBBBA, the CTC is $2,200 per child with a $1,700 refundable cap, and federal tax liability is $0 because the $32,200 standard deduction exceeds AGI, so the ACTC is the full $1,700. Most wrong answers made one of three mistakes: applying the expired pre-TCJA $1,000 ACTC, using the single-filer EITC phaseout threshold or a stale one instead of the MFJ threshold, or ignoring or misstating the $1,700 refundable cap."
us,scenario_032,free_school_meals_eligible,24,llm_error,state_local_rule,False,,"The household's income ($42,664, 1.56 times the poverty guideline) does exceed the federal NSLP 130% FPL free-meal limit, and it has no SNAP/TANF categorical eligibility. Minnesota, however, runs a universal free school meals program (in place since the 2023-24 school year), so PolicyEngine sets school_meal_tier to FREE for every K-12 child in the state regardless of income. Every model that answered stopped at the federal income test or the categorical-eligibility check and never applied Minnesota's universal free-meals override."
us,scenario_032,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_medicaid_eligible,15,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds back the non-taxable part of Social Security benefits. The Head's $13,664 in Social Security dependent benefits therefore counts in full alongside the $29,000 in wages. That puts household MAGI at $42,664, or 1.56 times the FPL for a 3-person household, above Minnesota's 138% expansion-adult limit. Minnesota's Medical Assistance limit for parents and caretakers is the same 133% FPL plus the 5% disregard, so the 50-year-old Head qualifies through no pathway. Most models counted only wages, or wages plus the taxable part of Social Security (about $30,916). The rest correctly found $42,664 above 138% but then invented a higher Medicaid limit for parents, or a larger household."
us,scenario_032,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific and requires the applicant to be pregnant, postpartum, breastfeeding, an infant, or a child under age five. A 50-year-old head with all relevant unlisted statuses set to false fails categorical eligibility regardless of household income or another household member's circumstances."
-us,scenario_032,payroll_tax,40,prompt_ambiguity,state_local_rule,False,,"The reference adds three pieces: federal Social Security ($1,798) and Medicare ($420.50) on the head's $29,000 in wages, and $127.60 of Minnesota Paid Leave (0.44% of $29,000). Minnesota Paid Leave premiums began in 2026 and are owed by the employer, which may deduct up to 0.44% of wages from the employee or pay that share itself. The reference counts the most the employer may deduct; the prompt does not say what this employer does, and counting only what the law requires of the employee gives federal tax alone, $2,218.50. Six models matched the reference, all adding $127.60. Thirty-three answered federal tax alone ($2,218.50, or $2,219 rounded): sixteen said no Minnesota employee payroll tax applies or none is mandatory, sixteen did not mention one, and one said paid-leave withholding is optional. One added the paid-leave premium at 0.35% ($2,320), and another recorded $2,320 against an explanation that totals $2,218.50. Of the rest, one submitted about double FICA ($4,428.60), one used an invented 5.24% rate ($1,520), one cut its own FICA total to $2,175.90 without a reason, and two returned no value. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_032,payroll_tax,41,prompt_ambiguity,state_local_rule,False,,"The reference adds Minnesota Paid Leave, which began January 1, 2026, to employee FICA ($1,798 Social Security plus $420.50 Medicare). The engine assumes the employer withholds the largest employee share it is allowed to: 0.44%, half of the 0.88% premium, on $29,000, which is $127.60. Minnesota lets the employer pay that share instead, and the prompt says nothing about this. So ""mandatory employee state payroll tax"" can mean (a) the employee's 0.44% share withheld, giving $2,346.10, or (b) nothing, because the employer is allowed to cover the whole premium, giving $2,218.50. Almost every model chose reading (b). The real errors are the few answers that are neither $2,218.50 nor $2,346.10: a wrong Paid Leave rate, invented rates or adjustments, an employer-share mix-up, and missing outputs. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_032,reduced_price_school_meals_eligible,26,llm_error,state_local_rule,False,,"Minnesota has universal free school meals, so PolicyEngine gives every student the FREE tier whatever the family's income. Reduced-price eligibility is false only because the better free tier replaces it. Income is not the reason: the household sits at 1.56 of the poverty guideline, inside the 130-185% reduced-price band. Every model that answered placed income in the federal 130-185% band and stopped there. None applied Minnesota's universal free-meals rule, which moves the student out of the reduced-price tier."
us,scenario_032,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,snap,8,llm_error,categorical_eligibility,False,,"SNAP eligibility ends at the gross-income screen for this three-person Minnesota household: $29,000 of wages plus $13,664 of Social Security dependent benefits produces $42,664 of annual gross income, above the applicable limit. Take-up assumptions and benefit-allotment calculations do not create eligibility after that screen fails, so the annual benefit is $0."
@@ -300,68 +302,68 @@ us,scenario_032,spouse_medicaid_eligible,17,llm_error,age_disability,False,,"The
us,scenario_032,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,spouse_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC does not cover every low-income woman of childbearing age or every adult living with a child. An adult woman must be pregnant, breastfeeding, or within the applicable postpartum period; the spouse has none of these statuses because all unlisted statuses are false, so she fails categorical eligibility regardless of the income test."
us,scenario_032,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Minnesota starts from federal AGI ($29,000 wages plus the $1,916 of Social Security made taxable by the §86 provisional-income formula = $30,916), then applies two Minnesota-specific subtractions the models ignored or undersized: the Minnesota Social Security benefit subtraction, which removes the entire $1,916 for a joint filer at this income (far below the phase-out range), and the Minnesota dependent exemption for Child 1 (roughly $5,300, indexed). That leaves $29,000 of Minnesota income against a 2026 joint standard deduction of roughly $30,700 plus the dependent exemption, so Minnesota taxable income is zero and tax before refundable credits is $0. Every wrong answer pairs a too-small joint standard deduction with a missing Social Security subtraction and a missing dependent exemption, manufacturing a few hundred to a few thousand dollars of phantom taxable income taxed at the 5.35% first-bracket rate."
-us,scenario_032,state_refundable_credits,46,llm_error,state_local_rule,False,,"Starting in 2023, Minnesota's Child and Working Families Credit replaced the old Working Family Credit, which was a share of the federal EITC. The new credit has two parts: a per-child credit, originally $1,750 and now inflation-indexed, and a working-family part equal to 4% of earnings, capped at roughly $365 per return. Both parts phase out together at 12% of income above a joint-filer threshold. This household's AGI is $30,916 ($29,000 in wages plus $1,916 of taxable Social Security), which is well below the 2026 joint-filer threshold. So the full indexed child credit plus the capped working-family part comes to $2,187.60. Most models went wrong in one of four ways: they used the repealed EITC-percentage schedule for the working-family part, left out one of the two parts, used unindexed 2023 amounts, or applied a phaseout that does not reach this AGI."
-us,scenario_033,federal_income_tax_before_refundable_credits,37,prompt_ambiguity,taxable_income_or_deductions,False,,"The correct 2026 computation includes the $243 state tax refund in gross income, which gives AGI of $83,451.23. It then subtracts $47,500 of deductions: the $32,200 MFJ standard deduction, two $1,650 aged additions, and the OBBBA $6,000 senior deduction for each spouse. That leaves taxable income of $35,951.23, taxed as $2,480 plus 12% of $11,151.23, or $3,818.15. The largest group of wrong answers either left out the $12,000 senior deduction or assumed the TCJA had expired, bringing back personal exemptions and the 15% bracket. Answers close to the reference went wrong in smaller ways. Some dropped the $243 refund. Others took deductions this household does not qualify for: auto-loan interest and a qualified-overtime deduction, even though the spouse's usual hours are 40 a week. Several models computed a figure and then submitted a different, unsupported number. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_032,state_refundable_credits,47,llm_error,state_local_rule,False,,"Starting in 2023, Minnesota's EITC-linked Working Family Credit was replaced by the Child and Working Families Credit (CWFC), which is one combined refundable credit. A child under 18 generates the per-child child amount, which is inflation-indexed for 2026. The working-family part is only 4% of earnings, capped at the base amount, with extra amounts only for qualifying children aged 18 to 24. AGI of $30,916 is below the married-filing-jointly phase-out threshold, so the full $2,187.60 applies. The wrong answers fall into four groups: they missed the CWFC entirely ($0), stacked an old-style EITC-percentage WFC worth $1,000–$2,000 on top of the child credit, used the unindexed 2023 amounts ($1,750 + $350 = $2,100), or applied a phase-out that does not apply here."
+us,scenario_033,federal_income_tax_before_refundable_credits,38,prompt_ambiguity,taxable_income_or_deductions,False,,"AGI is $83,451.23: $79,000 wages + $775 interest + the $243 state tax refund + $16,320 taxable Social Security (85% cap) − $11,400 SE loss − $1,487 HSA. Under post-OBBBA 2026 law, deductions are $47,500: the $32,200 MFJ standard deduction, 2 × $1,650 aged additions, and the $6,000-per-senior deduction for both spouses. That leaves $35,951.23 of taxable income, taxed at 10% up to $24,800 and 12% above it, for $3,818.15. The models mostly went wrong in four ways. Many left out the $12,000 senior deduction. Others assumed TCJA had expired, bringing back personal exemptions and the 15% bracket. Some dropped the $243 refund. Others claimed deductions that don't apply here: auto-loan interest, overtime, or invented itemized deductions. Several also made arithmetic slips or reported numbers that don't match their own working. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_033,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_033,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on each spouse's wages, producing $4,898 and $1,145.50 respectively and a total of $6,043.50. Both workers are below the Social Security wage base, their combined wages are below the married-filing-jointly Additional Medicare Tax threshold, and the self-employment loss belongs outside employee payroll tax. Most wrong answers correctly identified this base and these rates but then replaced their own exact result with an unsupported rounded or adjusted number."
+us,scenario_033,payroll_tax,11,llm_error,other,False,,"This is a simple calculation: employee FICA of 7.65% (6.2% Social Security plus 1.45% Medicare) on combined wages of $79,000 gives $6,043.50. Both spouses earn well under the Social Security wage base, the $250,000 joint Additional Medicare Tax threshold doesn't apply, South Dakota has no employee state payroll tax, and the negative self-employment income doesn't touch employee payroll tax. Almost every model either reached $6,043.50 and then submitted a different number (arbitrary rounding, an invented adjustment, or an unexplained replacement) or multiplied 7.65% × $79,000 wrong. One model gave no answer."
us,scenario_033,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Self-employment tax is a nonnegative liability assessed only on positive net earnings from self-employment. The listed negative SSTB amount does not create a negative self-employment tax or refundable tax offset, so the self-employment tax base and liability are both zero."
-us,scenario_036,federal_income_tax_before_refundable_credits,31,llm_error,thresholds_rates,False,,"The correct computation is short. Taxable income is $59,220 of pension income minus the 2026 single standard deduction of $16,100, which gives $43,120. The tax is 10% of $12,400 plus 12% of $30,720, which gives $4,926.40, and no nonrefundable credit applies. The shared trap is the 2026 parameter set under OBBBA (P.L. 119-21), which made the TCJA rate structure permanent and set the personal exemption to zero. Models went wrong in three ways. Many applied a TCJA sunset instead: an $8,300 standard deduction plus a personal exemption of about $5,300, with a 15% bracket. Others used a stale 2023, 2024 or 2025 standard deduction or bracket threshold. Several computed exactly $4,926 and then submitted a different number."
+us,scenario_036,federal_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"The calculation is simple: $59,220 of taxable pension income, minus the 2026 OBBBA single standard deduction of $16,100, leaves $43,120 of taxable income. Under the 2026 single brackets, tax is 10% of the first $12,400 ($1,240) plus 12% of the remaining $30,720 ($3,686.40), for $4,926.40, with no nonrefundable credits. The models went wrong in three main ways. Some assumed the TCJA would expire in 2026 and used an $8,300 standard deduction, a personal exemption and a 15% bracket, even though OBBBA made the TCJA rate and deduction structure permanent. Others used stale 2024 or 2025 deductions and brackets. Several did the arithmetic correctly, got about $4,926, and then submitted a different number anyway. A few added deductions or credits the facts don't support: mortgage interest from a balance alone, the elderly credit for a 56-year-old, or medical costs below the 7.5% AGI floor."
us,scenario_036,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid requires both membership in a qualifying eligibility category and satisfaction of that category's financial rules. This 56-year-old has medicaid_category NONE, and MAGI income of 3.71 times FPL exceeds the limits for all applicable MAGI pathways; the listed expenses and assets do not create another pathway."
us,scenario_036,payroll_tax,1,llm_error,payroll_tax_base,False,,"Taxable private pension income is not wages and is excluded from the employee Social Security, Medicare, and mandatory state payroll-tax bases. With no wages or other earned income listed, every employee-side payroll-tax component is zero."
us,scenario_036,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_036,state_income_tax_before_refundable_credits,30,llm_error,taxable_income_or_deductions,False,,"The reference starts from NJ gross income of $59,220 (all taxable pension), subtracts only the $1,000 single personal exemption to reach $58,220, and applies the NJ single schedule: $280 + $262.50 + $175 + 5.525% x $18,220 = $1,724.16. The head is 56 and not disabled, so NJ's pension/retirement exclusion (age 62+ or disabled) does not apply, and the reference allows no medical expense deduction. Twelve models subtracted a medical deduction above the 2% floor, several of them counting the $7,389 employer-sponsored premium. Six models wrongly excluded the whole pension or part of it despite the age test. The rest invented standard deductions or credits, used wrong bracket rates, or left out the $1,000 exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,31,llm_error,taxable_income_or_deductions,False,,"The head is 56 and not disabled, so New Jersey's pension exclusion (age 62+ or disabled only) does not apply and the full $59,220 is NJ gross income. The only subtraction is the $1,000 single personal exemption. No medical deduction survives: the $1,000 'health insurance premiums excluding Part B' is the total that already includes the $1,000 'other health insurance premiums', employer-sponsored premiums are not out-of-pocket deductible, and OTC items don't count, which leaves about $1,100 of qualifying expenses, below the 2% floor of $1,184.40. The tax on $58,220 is $280 + $262.50 + $175 + 5.525% × $18,220 = $1,724.16. The models fell into three groups: those that applied the age-62 pension exclusion, those that built a medical deduction by double-counting premiums or adding the $7,389 ESI premiums, and those that used invented brackets, standard deductions, or numbers that didn't follow from their own work."
us,scenario_036,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_037,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"The step that separates the reference from most wrong answers is the OBBBA qualified-overtime deduction. The full $9,629 FLSA overtime premium is deductible below the line, and it stacks on the $16,100 2026 single standard deduction. From AGI of $41,539 ($41,943 minus the $386 traditional 401(k) and the $18 IRA deduction), taxable income is $15,810, and tax is $1,240 + 12% × $3,410 = $1,649.25. Most models left out the overtime deduction, and many made other mistakes: they assumed the TCJA had sunset (personal exemptions and a 15% bracket), subtracted the ESI premiums from wages that were already stated, used 2024 or 2025 standard deductions and brackets, or claimed a Saver's Credit even though AGI of $41,539 is above the 2026 single limit."
+us,scenario_037,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The correct path starts with AGI of $41,539, which is wages minus the $386 traditional 401(k) and the $18 traditional IRA. The ESI premiums are not subtracted again. Taxable income then subtracts both the $16,100 2026 OBBBA standard deduction and the $9,629 qualified overtime deduction. The overtime deduction comes off below the line, so it does not lower AGI. That leaves $15,810 of taxable income, taxed at 10% on the first $12,400 and 12% on the rest, for $1,649.25. Most models missed the overtime deduction or assumed the TCJA sunset. Others subtracted ESI premiums from wages, used 2024/2025 brackets, or claimed a Saver's Credit, even though the $41,539 AGI is above the single filer's eligibility ceiling."
us,scenario_037,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A 44-year-old single filer with no dependent children or qualifying education expenses and $41,943 of wages receives neither EITC nor refundable CTC. All other refundable federal credit components evaluated to zero, so federal refundable credits total $0."
us,scenario_037,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"North Carolina’s Medicaid expansion does not make every adult aged 19–64 eligible; the expansion pathway applies an income limit. At 2.60 times FPL, the head exceeds the applicable MAGI threshold and qualifies through no other Medicaid category, so the eligibility category is NONE."
-us,scenario_037,payroll_tax,14,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 7.65% of the full $41,943 of gross wages, which already include the overtime: 6.2% Social Security ($2,600.49) plus 1.45% Medicare ($608.18), for $3,208.67. The facts list no cafeteria-plan salary reduction, so the $5,789 employer-sponsored insurance premium stays in the FICA wage base. The main trap was subtracting that premium as a pre-tax deduction, which eight models did to reach $36,154 and $2,765.78, about $442.89 too low. The other wrong models used the right base but then made arithmetic slips, added unexplained adjustments, rounded coarsely, or gave no answer."
-us,scenario_037,state_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"NC tax starts from federal AGI of $41,539.45, which is gross wages less only the traditional 401(k) ($386) and traditional IRA ($18) contributions. Subtracting the $12,750 single standard deduction leaves $28,789.45 of taxable income, and the 2026 flat 3.99% rate gives $1,148.70. The largest group of models also subtracted the $5,789 employer-sponsored insurance premiums from the stated gross wages, which gives an AGI of $35,750 and a tax of $917.70. A second group subtracted the $9,629 overtime premium, which is a federal below-the-line deduction that never reduces the AGI NC starts from. That gives $31,910 and about $764. The remaining models used outdated or invented standard deductions or rates, made up credits, or changed a correct $1,148.68 figure into a different final number."
-us,scenario_038,child1_chip_eligible,8,llm_error,health_coverage,False,,"CHIP is residual coverage: it reaches only children who fail Medicaid, so a CHIP determination requires a Medicaid floor test before any CHIP ceiling test. Household income of ~$22,992 for a family of four is roughly 71% FPL, far under Louisiana's Medicaid limit for children ages 6–18 (142% FPL), so 7-year-old child1 is Medicaid-eligible under the OLDER_CHILD category and CHIP eligibility is therefore 0. Every wrong model screened only against an upper LaCHIP ceiling (200–250% FPL), read ""income well below the ceiling"" as qualifying, and never applied the lower bound — the very fact that made the answer No."
+us,scenario_037,payroll_tax,15,llm_error,payroll_tax_base,False,,"The FICA base is the full $41,943 of gross wages, so 6.2% Social Security ($2,600.49) plus 1.45% Medicare ($608.18) gives $3,208.67. The household facts never say the $5,789 of employer-sponsored insurance premiums is paid through a Section 125 salary-reduction plan, and the prompt says unlisted status inputs are false. Even so, the largest group of models (eight) subtracted that amount from FICA wages and landed on $2,765.78. The other wrong answers started from the right 7.65% × gross-wages calculation and then broke it by changing the number arbitrarily, rounding loosely, giving a placeholder, or giving no answer at all."
+us,scenario_037,state_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI. AGI is gross wages of $41,943 minus the traditional 401(k) contribution ($386) and the traditional IRA deduction ($18), which gives $41,539.45. NC then subtracts its $12,750 single standard deduction and applies the flat 2026 rate of 3.99% to $28,789.45, for $1,148.70 with no nonrefundable credits. The most common mistake was subtracting the $5,789 employer-sponsored insurance premium from the listed gross wages, which gives an AGI of $35,750. Another group subtracted the $9,629 federal overtime-premium deduction, which comes off federal taxable income, not AGI, and so never reaches the NC base. Other models used the wrong rate (4–4.5% instead of 3.99%), used the wrong standard deduction ($10,750, $13,750, $15,400, $15,500 or $21,500), or made arbitrary final adjustments."
+us,scenario_038,child1_chip_eligible,9,llm_error,categorical_eligibility,False,,"Under PolicyEngine, CHIP only covers children who do not qualify for Medicaid, so CHIP is mutually exclusive with Medicaid. Child 1 is 7, and household income is about $22,992, roughly 70–73% FPL for a family of 4. That is well within Louisiana's Medicaid limit for children aged 6–18, so the engine puts the child in the OLDER_CHILD Medicaid category, and that Medicaid eligibility sets is_chip_eligible to False. Every model that answered checked only whether income was below a CHIP ceiling and marked the child CHIP-eligible. None of them checked first whether the child already qualified for Medicaid."
us,scenario_038,child1_head_start_eligible,3,llm_error,age_disability,False,,"Head Start for preschool-age children requires the child to fall within the program’s preschool age range; a seven-year-old does not satisfy that categorical requirement. Income eligibility cannot override the age restriction, so Child 1 is ineligible even under an income calculation that falls below a poverty guideline."
us,scenario_038,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Louisiana evaluates a 7-year-old under the Medicaid older-child category. The household's MAGI is 0.66 times FPL, below that category's 2026 income limit, so the child qualifies; the listed assets do not alter this MAGI eligibility determination."
us,scenario_038,child1_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five. Child 1 is age seven, so the child fails the categorical age requirement regardless of household income."
-us,scenario_038,child2_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP is residual to Medicaid: a child who qualifies for Medicaid cannot be CHIP-eligible, so the operative test for a 6-year-old is the Medicaid children's MAGI screen, not the CHIP ceiling. Household MAGI of roughly $22,993 for a family of four sits near 71% of FPL, far under the OLDER_CHILD (ages 6–18) Medicaid limit, so child2 is Medicaid-eligible and therefore excluded from CHIP. Every wrong model ran a single test — income versus a 202–250% FPL CHIP ceiling — found the household comfortably under it, and answered ""eligible,"" never screening the child against Medicaid first. Being far below the CHIP limit is exactly what makes the child Medicaid-eligible and thus CHIP-ineligible; the models read the same fact in the opposite direction."
+us,scenario_038,child2_chip_eligible,8,llm_error,categorical_eligibility,False,,"Under PolicyEngine's rules, CHIP and Medicaid don't overlap. CHIP covers only children who are not eligible for Medicaid. Household MAGI is about $23K for a family of four, roughly 70% of FPL. That is below Louisiana's Medicaid income limit for children aged 6–18, so Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore not CHIP-eligible. Every model compared income only to the upper CHIP ceiling and never checked whether the lower Medicaid pathway applied first."
us,scenario_038,child2_head_start_eligible,4,llm_error,categorical_eligibility,False,,"Head Start preschool eligibility requires satisfying the program’s applicable age and income rules; a six-year-old is outside the ordinary preschool-age category, and no fact establishes an exception such as delayed kindergarten entry. The household contains four people, and PolicyEngine’s traced income test also does not place this child within the qualifying threshold, so low income alone cannot produce eligibility."
us,scenario_038,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Louisiana evaluates the 6-year-old under the older-child Medicaid category using household MAGI relative to the applicable child income limit. The household's MAGI is 0.66 times FPL, so the child satisfies that category's income test; the listed assets do not disqualify a MAGI-category child."
us,scenario_038,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday, so a six-year-old is categorically ineligible regardless of household income. All four models incorrectly treated age six as falling within the WIC child category; the models that also evaluated income therefore never reached the correct result because they first misapplied the age rule."
us,scenario_038,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The requested amount cannot fall below zero: taxable income and income tax before credits are zero, and nonrefundable credits used are capped at the existing tax liability. Refundable credits are excluded from this output, so the resulting federal income tax before refundable credits is $0."
-us,scenario_038,federal_refundable_credits,40,llm_error,thresholds_rates,False,,"The reference is the sum of two credits. The first is the 2026 two-child EITC plateau maximum of $7,316: earned income of about $21,896 (wages plus SE income minus the deductible half of SE tax) sits above the phase-in end and below the MFJ phase-out start. The second is the refundable ACTC, equal to 15% × (earned income − $2,500) = $2,909.35. That amount binds below the $1,700-per-child ($3,400) refundable cap under the permanent post-OBBBA $2,200 CTC. The wrong answers fall into four groups: (1) assuming the TCJA lapsed, so the CTC reverted to $1,000 per child and the ACTC capped at $2,000; (2) granting the full $3,400/$3,600 refundable cap without the 15% earned-income phase-in; (3) projecting the wrong 2026 EITC maximum; (4) measuring earned income as gross SE income or as 92.35% of SE income instead of SE income minus half the SE tax."
+us,scenario_038,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"Earned income is $7,468 in wages plus $15,525 in self-employment income, minus half of the SE tax (about $1,097), which comes to about $21,896. That puts the couple filing jointly with two children on the 2026 EITC plateau: past the phase-in end of about $18,290 and below the joint phase-out start of about $31,000. So the EITC is the full 2026 maximum of $7,316. Under the OBBBA rules for 2026, the CTC is $2,200 per child and the refundable cap is $1,700 per child, so the refundable CTC is the smaller of $3,400 and 15% × (earned income − $2,500), which is $2,909.35. Models mostly missed in three ways: they used the wrong EITC maximum, they applied the pre-TCJA $1,000-per-child CTC with a $3,000 floor, or they mismeasured the ACTC's earned-income base, either by skipping the 15% limit or by calculating net SE earnings wrongly."
us,scenario_038,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The household qualifies for the free tier through two independent routes: SNAP receipt confers categorical eligibility, and school-meal income is 70% of the federal poverty guideline, below the 130% cutoff. The two K–12 children therefore generate positive annual free-school-meal support, so the binary output is Yes."
us,scenario_038,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_038,head_medicaid_eligible,3,llm_error,health_coverage,False,,"The trap is Louisiana's ACA adult expansion: LA adopted expansion effective July 1, 2016, so a 25-year-old non-disabled, non-pregnant adult qualifies through the MAGI ADULT category at up to 138% FPL, with no resource test. The household's countable MAGI of about $22,992 (wages $7,468 plus self-employment $15,525) measured against the four-person poverty line is 0.66 FPL — less than half the 138% ceiling — so the head clears the only test that applies. Every wrong model priced the head against a pre-expansion parent/caretaker standard (or an asset screen) instead of running the single MAGI-versus-138%-FPL comparison, and none of them actually computed the FPL ratio the record supplies."
+us,scenario_038,head_medicaid_eligible,4,llm_error,thresholds_rates,False,,"Louisiana has run the ACA adult expansion since 2016, so a 25-year-old non-dependent adult who does not receive SSI falls in the MAGI-based ADULT category, which has a 138% FPL limit. The head's household MAGI is about $22,992 from $7,468 in wages plus $15,525 in self-employment income. For a household of four that is 0.66x FPL, far below 138%, so the head is eligible. All four models said the income was above the limit. None of them compared household MAGI to the poverty guideline for four people, and some confused the expansion limit with Louisiana's much lower pre-expansion parent limit."
us,scenario_038,head_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC does not extend categorical eligibility to a parent or caretaker merely because children live in the household. The head is not specified as pregnant, postpartum, or breastfeeding, and the children are ages 7 and 6, above WIC's under-5 age limit, so income eligibility and adjunctive eligibility cannot make the head eligible."
us,scenario_038,payroll_tax,5,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes only employee-side FICA on the head’s wage earnings; self-employment income belongs in the separate self-employment-tax output. PolicyEngine applies the underlying wage amount and component-level calculations to produce $462.98 of Social Security tax plus $108.28 of Medicare tax, totaling $571.26."
us,scenario_038,reduced_price_school_meals_eligible,2,llm_error,thresholds_rates,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a general test of whether income is below 185% of the federal poverty guideline. At 70% of the guideline, and with categorical eligibility, this household receives the superior free-meals tier, so reduced-price support is zero."
-us,scenario_038,self_employment_tax,8,llm_error,payroll_tax_base,False,,"The SECA computation here is a single closed-form line with no branch points: net earnings from self-employment are $15,525 × 0.9235 = $14,337.3375, and because $7,468 of wages plus that base sit far below the Social Security wage base, the full 15.3% applies, giving exactly $2,193.61. The separating step is executing that arithmetic to the cent and stopping there — the 92.35% factor already embodies the employer-share equivalent, and the one-half-of-SE-tax deduction under §164(f) reduces AGI for income tax only, never the SECA base itself. Seven of the eight wrong answers name the right formula and then miss on execution: rounded or miscomputed multiplications ($2,191.73, $2,192, $2,194.96, $2,196.28, $2,197, $1,360.68) or a second, invalid reduction of the base by the half-SE-tax deduction ($2,054.08)."
-us,scenario_038,snap,38,llm_error,thresholds_rates,False,,"The right answer follows from the standard SNAP formula with FY2026 parameters. Monthly earnings of $1,916.04, from wages plus self-employment, get the 20% earned income deduction ($383.21) and the $223 standard deduction for four people. That leaves $1,310 net, with no shelter deduction because no rent, mortgage payment or utility costs are listed. Thirty percent of net is $393, and the $994 maximum allotment minus $393 gives $601 a month, or $7,212 a year. Most models set up this formula correctly but plugged in the wrong year's parameters: FY2025's $975/$204, guessed maximums of $1,000-$1,050, or a $879 maximum. Others made up shelter, dependent or medical deductions, applied 20% to annual income and then subtracted a monthly deduction, or wrongly declared the household ineligible."
+us,scenario_038,self_employment_tax,9,llm_error,payroll_tax_base,False,,"The calculation is a single step: net earnings from self-employment = $15,525 × 0.9235 = $14,337.34. That amount plus the $7,468 in wages is far below the Social Security wage base, so the full 15.3% rate applies and SE tax = $14,337.34 × 0.153 = $2,193.61. The deduction for half of SE tax lowers AGI only. It does not lower the SE tax base again, because the 92.35% factor already accounts for it. Most models used the right method but made arithmetic or transcription slips. One wrongly applied the half-SE-tax deduction a second time, and one gave no answer."
+us,scenario_038,snap,39,llm_error,thresholds_rates,False,,"The calculation is simple. Monthly earned income is $1,916.04. Subtract the 20% earned income deduction ($383.21) and the FY2026 four-person standard deduction of $223, which leaves net income of $1,310. 30% of that is $393, and subtracting it from the FY2026 maximum allotment of $994 gives $601 a month, or $7,212 a year. There are no shelter, dependent care or medical deductions, because no rent or mortgage payment is listed and no member is elderly or disabled. Most wrong answers came from using stale or guessed parameters, such as the FY2025 values of $975 and $204, a $1,000–$1,050 allotment, or a $198–$217 standard deduction. Others invented deductions, made arithmetic slips, or wrongly found the household ineligible."
us,scenario_038,spouse_chip_eligible,4,llm_error,health_coverage,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The spouse qualifies for Medicaid under Louisiana's OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility without regard to whether other coverage was listed."
us,scenario_038,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The trap is that the 18-year-old is labeled \""spouse,\"" and all three models let that label override the age-based pathway: PolicyEngine assigns her the OLDER_CHILD medicaid_category, which covers ages 6-18 in Louisiana and does not require being a tax dependent or an unmarried child. The income test is not close either — household MAGI is 0.66 x FPL, below Louisiana's ~142% FPL older-child limit and below the 138% FPL adult expansion limit, so both the child pathway and the adult pathway the models invoked return eligible. Every wrong answer therefore combines a category error (adult instead of older child) with an unperformed or inverted FPL comparison, and one model additionally imported a nonexistent asset test into a MAGI category."
us,scenario_038,spouse_wic_eligible,5,llm_error,categorical_eligibility,False,,"WIC income eligibility does not independently confer eligibility: a person must also be pregnant, breastfeeding, postpartum, an infant, or a child under age five. The spouse is 18, all unlisted statuses are false, and the only children are ages six and seven, so neither parenthood, childbearing age, nor low household income places the spouse in a WIC categorical group."
us,scenario_038,state_income_tax_before_refundable_credits,10,llm_error,state_local_rule,False,,"Louisiana's 2024 tax reform took effect in 2025. It replaced the graduated brackets and the old combined personal exemption–standard deduction ($9,000 MFJ, plus $1,000 per dependent) with a flat 3% rate and a $25,000 standard deduction for joint filers. Federal AGI is $21,896: $22,993 of wages and SE income minus the half-SE-tax deduction of about $1,097. That is below the $25,000 MFJ deduction, so Louisiana taxable income is zero and so is the tax. Most models used pre-reform exemptions and brackets (1.85%, 2%/4%) or applied only the $12,500 single-filer deduction to a joint return, which left positive taxable income. One model reached $0 in its reasoning but submitted a different number."
-us,scenario_038,state_refundable_credits,36,llm_error,state_local_rule,False,,"Louisiana has a refundable state EITC equal to 5% of the federal EITC. This couple files jointly with two children, and their earned income is about $21,900 (wages plus self-employment income, minus half of the SE tax). That puts them on the federal EITC plateau at the 2026 two-child maximum of $7,316, so the LA credit is 0.05 x $7,316 = $365.80. Half of the models (18 of 36) said Louisiana has no refundable EITC, or that it is nonrefundable, and answered $0. Most of the others used the 5% rate but misstated the federal EITC maximum. A few used the wrong state rate (3.5% or 50%), and two badly misplaced the household on the federal EITC schedule."
-us,scenario_039,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's gross-income source list leaves out estate_income, while its qualified-business-income sources include it. As a result, the $25,950 is excluded from AGI and from the §86 provisional income that sets taxable Social Security (AGI $32,699.07 = $10,129.10 + $26,800 + $2,030 − $6,260.03), yet it still generates a $3,319.81 §199A deduction. That conflicts with IRC §61(a)(15)/§662(a), which tax estate distributions to the beneficiary, and with §199A(c)(3)(A)(ii), which counts an item as QBI only if it is included in taxable income.","The reference files the head as single, because no dependent child is listed. It leaves the $25,950 estate income out of AGI and out of Social Security provisional income, so taxable SS is $10,129.10 and AGI is $32,699.07. It then subtracts the $16,100 standard deduction and a $3,319.81 QBI deduction and taxes $13,279.25 at 10%/12%, giving $1,345.51. Every model that answered except glm-5.3 put the estate income into AGI, which pushed SS to the 85% cap and AGI to about $79,209; many also used joint qualifying-surviving-spouse status, TCJA-sunset parameters, or stale brackets. The models that used single status and correct 2026 parameters all got $8,596.04, and that entire gap comes from how the estate income and QBI deduction are handled. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 1,345.51; the corrected value is 8,596.03. PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))."
+us,scenario_038,state_refundable_credits,37,llm_error,state_local_rule,False,,"Louisiana's EITC equals 5% of the federal EITC and is refundable. In this case it is the only state refundable credit. The household files jointly with two qualifying children and has earned income of about $21,900 (wages of $7,468 plus net self-employment earnings after the SE-tax deduction). That puts it on the federal EITC plateau, well below the MFJ phase-out start of about $31k, so the federal credit is the 2026 two-child maximum of $7,316, and 5% of that is $365.80. Most wrong models said Louisiana has no refundable EITC, or that its EITC is nonrefundable. The rest used the right structure but the wrong federal maximum, the wrong state percentage (3.5% or 50%), or wrongly applied the phase-out."
+us,scenario_039,federal_income_tax_before_refundable_credits,43,llm_error,household_unit_or_filing_status,False,,"The case turns on filing status. Qualifying-surviving-spouse status under IRC §2(a) requires keeping a home for a dependent child. No dependent is listed, so the 'surviving spouse' flag does not unlock joint rates, and the head files single. That gives 85% of Social Security taxable under the single thresholds ($30,689.25), AGI of $79,209.22, the 2026 OBBBA single standard deduction of $16,100, and taxable income of $63,109.22. Single brackets (10% to $12,400, 12% to $50,400, 22% above) then produce $8,596.03. Most wrong answers used qualifying-surviving-spouse joint brackets and the $32,200 joint deduction. A second group assumed the TCJA sunset (which OBBBA repealed) or used stale 2025 deductions and brackets. A few invented credits or extra deductions that zeroed out the tax."
us,scenario_039,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_039,head_medicaid_eligible,3,llm_error,health_coverage,False,,"Every Virginia Medicaid pathway open to a 61-year-old is income-tested: the ACA expansion adult group caps MAGI at 138% FPL, and the SSI-linked aged/blind/disabled group sits far below that and keys off SSI receipt, which is $0 here. This head's Medicaid MAGI counts the full $36,105 Social Security benefit (untaxed portion included) plus $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, offset only by a $6,260 self-employment loss, landing at 368% FPL — 2.7x the expansion limit — so the engine assigns medicaid_category NONE. All three models either asserted without arithmetic that MAGI fell under 138% FPL or treated the ""is disabled"" flag as a categorical pass that skips the income test entirely. Disability and the expansion group's waived asset test change which income test applies, never whether one applies."
-us,scenario_039,head_medicare_eligible,19,llm_error,age_disability,False,,"A person under 65 qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or with ESRD or ALS. A disability flag on its own does not qualify anyone. The head is 61, and their only Social Security income is retirement income. No SSDI income or SSDI months are listed, so both count as 0, and ESRD and ALS default to false. None of the qualifying pathways applies, so the head is not eligible. All 19 models treated ""is disabled"" as automatic Medicare eligibility, and several also mistook the Social Security retirement income for SSDI."
+us,scenario_039,head_medicare_eligible,20,llm_error,age_disability,False,,"Before age 65, Medicare eligibility through disability requires 24 months of receiving Social Security Disability Insurance. Being flagged as disabled is not enough. PolicyEngine checks age ≥ 65, at least 24 months of SSDI receipt, or end-stage renal disease (ESRD). The head is 61, has no listed SSDI months (unlisted inputs count as 0), and has no ESRD. Their $36,105 of Social Security is retirement income, not disability benefits. Every model treated the disabled flag by itself as enough for Medicare, and that is why they answered 1 instead of 0."
us,scenario_039,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary income, not estate income, Social Security retirement benefits, IRA distributions, private pensions, or self-employment income. No wages were listed, and the prompt requires unlisted numeric inputs to be zero, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax bases are all zero."
us,scenario_039,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_039,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's federal gross-income sources omit the estate_income variable. That drops the $25,950 of estate/trust income from AGI, even though IRC §61(a)(15) and §662(a) include it: $32,699.07 = 26,800 + 2,030 − 6,260 + 10,129.10 taxable Social Security. With it included, taxable Social Security is $30,689.25, Virginia AGI is $48,520, Virginia taxable income is $38,840, and tax is $1,975.80.","The reference's federal AGI of $32,699.07 is IRA $26,800 + pension $2,030 − self-employment loss $6,260 + taxable Social Security $10,129.10. The $25,950 of estate income never enters AGI, which also cuts provisional income enough to reduce taxable Social Security from $30,689 to $10,129. Virginia then subtracts all taxable Social Security, applies the single $8,750 standard deduction (a surviving spouse files as single in Virginia) and one $930 exemption, and taxes $12,889.97 for $514.50. Nineteen models included estate income and landed between $1,969 and $2,029, or exactly $1,975.80 with 2026 parameters. The rest also used the married-filing-jointly deduction, skipped the Social Security subtraction, or invented age or disability relief. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 514.50; the corrected value is 1,975.80. PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))."
+us,scenario_039,state_income_tax_before_refundable_credits,44,llm_error,thresholds_rates,False,,"Virginia AGI is $48,519.97: federal AGI of $79,209.22 minus the $30,689.25 of federally taxable Social Security. Because the head has no dependent, a surviving spouse files as single and gets the 2026 single standard deduction of $8,750 plus one $930 exemption. That leaves taxable income of $38,839.97, and the tax is $720 + 5.75% × $21,839.97 = $1,975.80. Most models made the Social Security subtraction correctly but missed on the deduction. Some used the old $8,500 or $8,000 amount, which gives $1,990 or $2,019. Others gave the surviving spouse the joint $17,000–$17,500 deduction, which gives about $1,473. A smaller group skipped the Social Security subtraction or invented age, disability or retirement exclusions that don't apply to a 61-year-old with no disability income."
us,scenario_039,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_040,federal_income_tax_before_refundable_credits,9,llm_error,taxable_income_or_deductions,False,,"Every model got the same AGI the engine used ($28,000 wages + $4,300 taxable Social Security = $32,300, since workers' compensation is excluded and provisional income of $40,600 falls in the 50% tier between the $32,000 and $44,000 MFJ thresholds) or missed it, and then lost the case on the deduction side. For tax year 2026 the TCJA structure is permanent: personal exemptions remain repealed, the MFJ standard deduction is $32,200, the aged additional standard deduction is $1,650 per 65+ spouse ($3,300), and the OBBBA senior deduction adds $6,000 per spouse aged 65+ ($12,000) for a total of $47,500 — well above the $32,300 AGI, so taxable income and tax are $0. Six of the nine models instead applied a TCJA-sunset 2026 with a ~$16,500–$20,000 standard deduction plus revived personal exemptions of ~$10,200–$10,600, a combination that leaves a small positive taxable income taxed at 10%. The two that used a post-OBBBA standard deduction still failed by inflating includible Social Security or by treating the senior deduction as a tax offset rather than a $6,000-per-spouse deduction."
us,scenario_040,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_040,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Arizona Medicaid expansion does not cover the head through the ordinary adult MAGI pathway because the head is over 65, and the head qualifies through none of the aged, blind, or disabled pathways. At 2.46 times FPL, with no SSI, no disability, and no dependent status, the head has Medicaid category NONE and is ineligible."
-us,scenario_040,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee payroll tax applies separately to both spouses' wages and equals $1,736 of Social Security tax plus $406 of Medicare tax. No Additional Medicare Tax or mandatory Arizona employee payroll tax applies, so the household total is $2,142."
+us,scenario_040,payroll_tax,6,llm_error,other,False,,"Employee payroll tax here is simple FICA on each spouse's wages: 6.2% Social Security plus 1.45% Medicare on $20,000 and $8,000, which gives $1,530 + $612 = $2,142. Social Security benefits and workers' compensation are not wages for FICA. Arizona has no mandatory employee state payroll tax, and Additional Medicare Tax doesn't apply because combined wages are far below $250,000. Nearly every model worked out $2,142 in its own reasoning, then submitted a different number, so the miss came from the final answer contradicting the model's own correct work, not from a wrong rule."
us,scenario_040,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_040,snap,1,llm_error,taxable_income_or_deductions,False,,"The elderly or disabled household exception removes the SNAP gross-income test but does not establish eligibility; the household must still pass the net-income test after allowable deductions. This household's $56,400 annual income remains too high after the applicable deductions, so it receives no minimum allotment because minimum benefits apply only after eligibility is established."
-us,scenario_040,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Age 69 and disability establish potential aged/disabled categorical status, not Medicaid eligibility by themselves. The spouse qualifies through no Arizona Medicaid pathway: the engine assigns medicaid_category NONE, and the spouse’s MAGI income is 2.46 times FPL for MAGI-based categories. The models converted aged/disabled status into eligibility without completing the applicable income and resource tests."
+us,scenario_040,spouse_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"The spouse is 69, so Arizona's expansion adult group (ages 19-64) does not apply. The only route left is the aged/blind/disabled category, which has an income limit of 100% FPL, and that test uses the married couple's combined income, not just the spouse's own. The couple has $28,000 in wages plus $25,200 in Social Security, about $53,200 in total. That puts them at 2.46x the two-person FPL, far above the limit, so the engine assigns medicaid_category NONE. Every model treated disability or age as close to automatic eligibility, looked at the spouse's income alone, or ignored the income and asset limits it had just named."
us,scenario_040,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility applies at age 65, so the 69-year-old spouse qualifies through the age pathway without needing disability-based eligibility or 24 months of SSDI receipt. The spouse’s workers’ compensation and disability facts do not alter that age-based result."
us,scenario_040,state_income_tax_before_refundable_credits,6,llm_error,taxable_income_or_deductions,False,,"The case turns on two Arizona-specific rules: Arizona subtracts from federal AGI all Social Security benefits included there, and Arizona's standard deduction conforms to the base federal married-filing-jointly amount (about $32,200 for 2026) with no additional age-65 amount. Removing the $4,300 of taxable Social Security leaves $28,000 of wages as Arizona AGI, which is more than $4,000 below the standard deduction, so taxable income is zero and the 2.5% flat tax yields nothing before refundable credits; itemizing is worse at $16,324 of mortgage interest plus real estate taxes. The wrong answers all manufacture a positive base either by shrinking the deduction — inventing $20,000, $22,000, or an unstated ~$22,700 figure, in two cases by grafting a federal-style age-65 add-on Arizona does not have — or by leaving Social Security in the Arizona base. One model produced the zero result and then overrode it, and one returned nothing at all."
us,scenario_040,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"Arizona’s refundable credits are not awarded merely because a married filer is elderly and has income below a general threshold. The household satisfies no applicable Arizona refundable-credit eligibility pathway in 2026, so az_refundable_credits and the aggregate state_refundable_credits are both zero; the other failure is a missing required output."
-us,scenario_042,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"A widow with no dependent child files as Single, not Qualifying Surviving Spouse. Under OBBBA's permanent TCJA parameters for 2026, the Single deduction is $16,100 plus the $2,050 aged add-on, plus the $6,000 senior deduction, for $24,150 in total. Taxable Social Security is $16,846.25 under the 25k/34k provisional-income formula, which makes AGI $50,580.71. Of the $26,430.71 in taxable income, the $4,681 of qualified dividends and capital gain falls in the 0% band, and the $21,749.71 of ordinary income is taxed at 10%/12% (1,240 + 12% × 9,349.71 = $2,361.96). The models split into three main error groups: using QSS/joint parameters, which makes the answer too low; leaving out the $6,000 senior deduction or applying expired pre-TCJA law, which makes it too high; and getting the Social Security taxable amount wrong. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)."
+us,scenario_042,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The head is a widowed person with no dependent child, so they file as Single, not Qualifying Surviving Spouse. Taxable Social Security uses the single $25k/$34k thresholds: provisional income of $48,525 gives $16,846.25 taxable and AGI of $50,580.71. Under 2026 OBBBA law (TCJA made permanent), the deductions are the $16,100 standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, for $24,150 total. Taxable income is $26,430.71, of which $4,681 of qualified dividends and capital gains is taxed at 0% and $21,749.71 is taxed at 10%/12%, giving $2,361.96. Most wrong answers came from one of three mistakes: using QSS joint parameters, leaving out the $6,000 senior deduction, or applying the pre-TCJA sunset schedule with personal exemptions. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)."
us,scenario_042,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_042,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax uses employee wages as its base, and the prompt lists no wage or salary income. The pension, IRA distributions, investment income, Social Security benefits, and farm loss do not generate employee-side Social Security or Medicare tax, so employee payroll tax is $0."
us,scenario_042,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_042,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,"PolicyEngine's Wisconsin module does not apply the retirement income exclusion created by 2025 Wisconsin Act 15 (up to $24,000 of retirement-plan/IRA/pension income for filers age 67+, effective tax year 2025): the trace subtracts exactly the $13,655.95 of taxable Social Security and nothing more, so the 79-year-old head's $24,086 of taxable IRA and pension income stays in WI AGI. Separately, the engine's non-Social-Security AGI ($29,981.71) is about $3,824 below the listed income items, which suggests the $3,753 of non-Schedule-D capital gain distributions was left out of federal AGI.","PolicyEngine subtracts exactly the $13,655.95 of taxable Social Security from $43,637.65 of federal AGI. That leaves $29,981.71 of WI AGI, with all $24,086 of IRA and pension income still taxed. It then applies the single-filer sliding-scale standard deduction of $12,671.36 and $950 of exemptions, taxes $16,360.34 at 3.5%/4.4% for $584.74, and subtracts the $300 school property tax credit to reach $284.74. One group of models zeroed the tax with a $24,000 age-67+ retirement income exclusion or a blanket pension exclusion, which the engine does not apply. The other group kept retirement income taxable but made one or more other errors: they inflated WI AGI with the $3,753 of capital gain distributions, mis-sized or skipped the standard deduction and exemptions, treated the surviving spouse as a joint filer, used wrong bracket rates, or left out or over-sized the $300 school property tax credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)."
+us,scenario_042,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,"The reference federal AGI of $43,637.65 is exactly $29,981.71 of non-Social Security income plus $13,655.95 of taxable Social Security, where 0.85×($29,981.71+$14,790−$34,000)+$4,500 = $13,655.95; that sum can only come out this way if PolicyEngine left the $3,753 of non-Schedule-D capital gain distributions out of gross income, which conflicts with IRC §61(a)(3) and Form 1040 line 7. Separately, the Wisconsin subtractions leave out the 2025 Wisconsin Act 15 retirement income exclusion for filers age 67 and older (up to $24,000), which would cover almost all of this filer's $24,086 of IRA and pension income.","The reference gets Wisconsin AGI of $29,981.71 by subtracting only the $13,655.95 of taxable Social Security from a federal AGI that contains none of the $3,753 in capital gains, and it applies no retirement-income subtraction and no 30% capital-gain subtraction. It then takes the single sliding-scale standard deduction ($12,671.36) and $950 of exemptions, taxes $16,360.34 at 3.5%/4.4% to get $584.74, and subtracts the $300 school property tax credit (12% of the first $2,500 of property tax). The wrong answers fall into three groups. Models that applied the $24,000 age-67+ retirement exclusion got $0. Models that kept the $3,753 of gains (less the 30% long-term exclusion) and otherwise followed the same steps got about $415–$430. The rest misapplied the standard deduction, filing status, exemptions, rates, or the $300 credit (skipping it, overstating it, or inventing other credits). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)."
us,scenario_042,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"The Wisconsin homestead credit requires qualifying household income and property-tax circumstances to be computed from the stated annual inputs under the 2026 rules. The household receives substantial Social Security, IRA, pension, dividend, interest, and capital-gain income, so reducing household income to roughly $14,420 and applying a maximum-credit shortcut creates a refundable credit that the full eligibility computation does not allow."
us,scenario_043,federal_refundable_credits,4,llm_error,age_disability,False,,"The single trap is the upper age limit on the childless EITC: IRC §32(c)(1)(A)(ii)(II) allows the no-qualifying-children credit only to filers at least 25 and under 65 at the close of the tax year, and the American Rescue Plan's one-year suspension of that ceiling applied to 2021 only. The head is 66 in tax year 2026, so the EITC is $0 regardless of the $4,088 in earnings sitting squarely in the phase-in range, and with no children, no education expenses, and no Recovery Rebate or refundable payroll tax credit, total federal refundable credits are $0. All four models jumped straight to the phase-in arithmetic on earned income and never applied the age test, three of them landing on 7.65% × ~$4,088 ≈ $312–313 and one on a fabricated 15.3% rate."
us,scenario_043,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -370,18 +372,18 @@ us,scenario_043,head_medicaid_eligible,9,llm_error,categorical_eligibility,False
us,scenario_043,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_043,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_043,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_043,payroll_tax,30,prompt_ambiguity,state_local_rule,False,,"The reference is $330.73: employee Social Security and Medicare, 7.65% of the head's $4,088.15 of wages, or $312.74, plus $17.99 of Colorado FAMLI (0.44% of wages). The prompt lists the wages as $4,088, on which 7.65% is $312.73. Colorado's FAMLI premium is 0.88% of wages in 2026; the employer remits it and may deduct up to half, 0.44%, from the employee or pay that share itself. The reference counts the most the employer may deduct; the prompt does not say what this employer does, and counting only what the law requires of the employee gives federal tax alone, $312.74. Sixteen models matched the reference within $1: seven at 0.44% ($330.72) and nine at the 2025 rate of 0.45% ($331.13). Twenty-six answered federal tax alone (about $312.73): fourteen said Colorado has no employee payroll tax or none is mandatory, eleven did not mention FAMLI, and one computed FAMLI and then left it out as like unemployment insurance. Of the other four, one reached $331.14 with FAMLI at 0.45% but submitted $315.14, one priced FAMLI at $24.53 (0.6% of wages), one subtracted the $8,389 of employer-sponsored insurance premiums from wages and returned $0, and one returned $0 with no calculation. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_043,payroll_tax,31,prompt_ambiguity,state_local_rule,False,,"Federal FICA on $4,088 is $312.74 ($253.47 Social Security plus $59.28 Medicare). The rest of the reference is Colorado's FAMLI employee premium: 0.44% (half of the 0.88% 2026 premium) × $4,088.15 = $17.99, for a total of $330.73. The prompt asks for 'mandatory employee state payroll taxes', and it can be read two ways. Under one reading, the FAMLI employee share counts because the law lets the employer withhold it from wages, which is how PolicyEngine treats it. Under the other, it is excluded because Colorado lets the employer pay that share instead, and the prompt never says which the employer does. Almost every model chose the second reading and landed on about $312.73. The few others made genuine errors: a wrong FAMLI rate, an arithmetic slip, or wrongly cutting FICA wages by health premiums. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_043,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_043,snap,32,llm_error,thresholds_rates,False,,"This one-person Colorado household is eligible and gets the maximum benefit. Its 66-year-old head counts as elderly, so the excess medical deduction, the standard deduction and the 20% earned-income deduction bring net income to $0. The $2,800 in assets is under the elderly limit, and the household is also categorically eligible through Colorado's TANF non-cash eligibility test. The benefit is the 2026 one-person maximum allotment for the contiguous states, $298 a month, or $3,576 a year. Most models got the eligibility right and then used a stale or guessed maximum ($291 for FY2024, $292 for FY2025, or $299-$328). A smaller group wrongly said the household fails the asset or income tests and answered $0."
+us,scenario_043,snap,33,llm_error,thresholds_rates,False,,"This one-person elderly Colorado household has monthly gross income of $387.35. The deductions bring net income to $0: 20% of earned income ($68.14), the $209 standard deduction, and the excess medical deduction for an elderly member ($5,000/12 − $35 = $381.67). With net income at $0, SNAP pays the full one-person contiguous-states maximum allotment, which is $298/month, or $3,576 for the year. Assets are not a barrier: Colorado's broad-based categorical eligibility applies, and $2,800 is also under the elderly asset limit. Most models did reach net income of $0 but then used the wrong maximum allotment (often $291, $292 or ~$300). Others wrongly said the household fails the asset test, or cut the benefit below the maximum for no reason."
us,scenario_043,ssi,4,llm_error,period_annualization,False,,"SSI eligibility is determined for the individual, not by subtracting annualized countable income from twelve times a monthly federal benefit rate. The head’s approximately $4,642 of individual income exceeds the applicable SSI income limit, so the income-eligibility gate fails and no SSI is payable despite aged categorical eligibility and assumed take-up."
-us,scenario_043,state_refundable_credits,46,llm_error,state_local_rule,False,,"The Colorado sales tax refund (the TABOR refund) is the only credit that pays anything here. PolicyEngine counts it as a refundable state income tax credit, and this 66-year-old full-year resident filer qualifies. At modified AGI of $4,625.40 it pays $19 for 2026. Every other Colorado refundable credit is zero: the head has no children, and at 66 is too old for the federal childless EITC, so the Colorado EITC is also zero. Most models checked only the EITC and child credits and never included the sales tax refund. A few invented senior or EITC amounts, or took the TABOR amount from the wrong tier."
+us,scenario_043,state_refundable_credits,7,llm_error,state_local_rule,False,,"Colorado's refundable credits available to this filer depend on two things. The TABOR surplus mechanisms (the sales tax refund and the senior housing credit) pay only when the state has excess revenue. The Colorado EITC is a percentage of the federal EITC. Forecasts showed FY2025-26 below the Referendum C cap, so no TABOR-funded refund exists for tax year 2026. The head is 66, above the federal childless-EITC age ceiling of 64, so there is no federal EITC for Colorado to match. Every wrong answer either paid a TABOR-contingent refund that is not triggered in 2026 or matched a federal EITC the head cannot claim because of age."
us,scenario_043,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_044,federal_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"The couple's only income is $56,806 of Social Security. Their provisional income is 50% of benefits plus other income: 0.5 × $56,806 + $0 = $28,403. That is below the $32,000 married-filing-jointly base amount under IRC §86, so none of the benefits are taxable. AGI and taxable income are $0, and so is regular income tax. Every wrong model skipped this test. They treated the benefits as taxable in full, assumed the 85% maximum inclusion applied automatically, or wrongly said the threshold was exceeded. Each then put a positive tax on income that is excluded by statute."
us,scenario_044,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered earned compensation, not Social Security retirement benefits. Because the household has no reported wages or self-employment income, every employee payroll-tax component has a zero base and payroll tax is $0."
us,scenario_044,spouse_medicare_eligible,1,parse_contract_failure,other,False,,"The spouse is age 61 and has no listed disability or other Medicare-qualifying status, so the spouse is not Medicare eligible in 2026. The model’s explanation correctly derives 0, but its submitted numeric value is 1, creating an internally inconsistent contract output."
us,scenario_044,state_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"Kansas subtracts federally taxable Social Security benefits from Kansas adjusted gross income, so the household’s only reported income does not enter the Kansas income-tax base. With no remaining Kansas taxable income, state income tax before refundable credits is $0; applying Kansas deductions, exemptions, brackets, or credits to the $56,806 benefit is the wrong computation path."
us,scenario_044,state_refundable_credits,3,llm_error,state_local_rule,False,,"All three models awarded the Kansas food sales tax credit of $125 per exemption (2 x $125 = $250). Kansas cut its state sales tax on food to 0% on January 1, 2025, and ended that refundable credit after tax year 2024, so it does not exist in tax year 2026. Kansas's other refundable credit, the state EITC (17% of the federal EITC), is also $0 because the household has no earned income: its only income is Social Security. That leaves Kansas refundable credits, and state_refundable_credits, at $0."
-us,scenario_045,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"The reference is a plain single-filer computation. AGI equals the stated gross wages of $36,275.59. The $21,208 of employer-sponsored insurance premiums is not subtracted from the listed gross wages and is not an itemized medical expense, so itemized deductions are only $1,541.71. The 2026 standard deduction under the permanent TCJA/OBBBA schedule is $16,100, which leaves $20,175.59 of taxable income. At 10% up to $12,400 and 12% above, that gives $1,240 + $933.07 = $2,173.07, and no nonrefundable credits apply. The wrong answers fall into three groups. Some subtracted the ESI premiums from wages, which drives taxable income to near zero. Some used a stale or sunset-era 2026 schedule: 2025 amounts ($15,000–$15,750 and $11,925), or pre-TCJA personal exemptions with a 15% bracket. The rest derived $2,173 or something close and then submitted an unrelated number."
+us,scenario_045,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"The correct calculation is simple. AGI equals the full $36,275.59 in gross wages. Subtract the post-OBBBA 2026 single standard deduction of $16,100 to get taxable income of $20,175.59. Tax is 10% on the first $12,400 plus 12% on the remaining $7,775.59, or $2,173.07, and no nonrefundable credits apply. The models failed in four main ways: (1) they subtracted the $21,208 employer-sponsored insurance premiums from the gross wages; (2) they assumed the TCJA expired, bringing back personal exemptions and the 15% bracket, even though OBBBA made TCJA permanent; (3) they used outdated 2024/2025 standard deductions and bracket thresholds; or (4) they correctly calculated about $2,173 but then submitted an unrelated number, or reduced the tax to zero with credits that don't exist or with the refundable EITC."
us,scenario_045,federal_refundable_credits,10,llm_error,credit_phaseout,False,,"The full $36,276 of annual gross wages enters the childless EITC income test; the separately listed employer-sponsored insurance premiums do not reduce earned income or AGI because no pre-tax payroll treatment is stated. At that income, the childless EITC is fully phased out, and the household has no qualifying child or other basis for a refundable federal credit."
us,scenario_045,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_045,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -391,25 +393,25 @@ us,scenario_045,payroll_tax,10,llm_error,payroll_tax_base,False,,"The trap is th
us,scenario_045,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_045,snap,4,llm_error,categorical_eligibility,False,,"This one-person household has no elderly or disabled member, so it must either pass the 130% FPL gross income test or qualify through Michigan's broad-based categorical eligibility (BBCE), which has a 200% FPL gross limit. Michigan counts child support paid in gross income and deducts it only when computing net income (7 CFR 273.9(d)(5)). That leaves gross income at the full $3,022.97 a month, about 232% of the guideline, which fails both limits, so SNAP is $0 for the year. The wrong answers came from three errors: taking child support or health premiums out of gross income, still paying a benefit after finding the household over the limit, and a submitted value that contradicts the model's own finding that the household is ineligible."
us,scenario_045,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_045,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Michigan tax for this filer is the flat 4.25% rate applied to AGI equal to full wages ($36,275.59) less the 2026 personal exemption of $5,900, which gives $1,290.96. Michigan has no standard deduction for a 44-year-old, and no nonrefundable credit applies here. Most models used the right structure but the wrong exemption: $5,000, $5,200, $5,600, $5,800, $6,000 or $7,350 in place of $5,900. A second group subtracted the $21,208 employer-sponsored insurance premium input from wages, which cut AGI to $15,068. The rest invented a standard deduction, used a 4.05% rate, treated the refundable Homestead Property Tax Credit as nonrefundable, or submitted a number different from their own computation."
-us,scenario_045,state_refundable_credits,42,llm_error,state_local_rule,False,,"The whole $760.79 is Michigan's refundable Homestead Property Tax Credit for a renter. The steps are: take 23% of the $10,560 rent as property tax ($2,428.80), subtract 3.2% of total household resources of about $36,276 ($1,160.82), and multiply the $1,267.98 excess by 60%. Federal EITC, and so Michigan EITC, is $0 at this income. About half of the models either left out the homestead credit (they checked only the EITC, or treated renters or the credit as ineligible or nonrefundable) or computed it incorrectly. The errors were using 20%, 17% or no rent share instead of 23%, using a 3.5% threshold, using 80% or no 60% multiplier, subtracting the $21,208 ESI premiums or the $5,200 child support from household resources, or adding a Michigan EITC that does not exist here."
+us,scenario_045,state_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"Michigan's tax is a flat 4.25% applied to federal AGI ($36,275.59, the full wage amount; the stated employer-sponsored insurance premiums are not subtracted again) minus the 2026 personal exemption of $5,900. That gives $30,375.59 × 0.0425 = $1,290.96, with no nonrefundable credits. Most wrong answers came from projecting the exemption from memory ($5,000, $5,200, $5,600, $5,800, $6,000 or $6,200) instead of using $5,900. A second group subtracted the $21,208 ESI premiums from AGI and landed near $390–$402. A third group invented a standard deduction or treated the refundable Homestead Property Tax Credit as nonrefundable and reported $0 or close to it. The rest computed a figure near the right one and then submitted an unrelated number."
+us,scenario_045,state_refundable_credits,43,llm_error,state_local_rule,False,,"The whole $760.79 is Michigan's refundable Homestead Property Tax Credit for a renter, with no state EITC. The correct steps are: count 23% of the $10,560 rent as property tax ($2,428.80). Subtract 3.2% of total household resources, which equal the full $36,275.59 in wages, giving $1,160.82 and a remainder of $1,267.98. Then apply the 60% non-senior rate to get $760.79. Most wrong answers left out the renter credit entirely and only looked at the zero Michigan EITC. The rest changed one input of the formula: they used 20% (or 17%) of rent instead of 23%, a 3.5% threshold instead of 3.2%, or a different credit rate. Others subtracted the $21,208 employer insurance premiums or the $5,200 child support from household resources, or added a federal EITC that a childless filer at $36k cannot get."
us,scenario_046,child1_chip_eligible,1,llm_error,state_local_rule,False,,"Oklahoma covers school-age children through Medicaid (SoonerCare), not through a separate CHIP. The engine therefore gives a 10-year-old in Oklahoma no CHIP pathway on age grounds. The household's MAGI of about $92,003 (wages plus interest) is also far above any Oklahoma children's income limit for a family of four. A wrong Yes comes from applying a generic, high CHIP income limit and understating MAGI, instead of Oklahoma's own CHIP rules for this child's age and income."
us,scenario_046,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A dependent child qualifies for Oklahoma Medicaid only through an applicable eligibility category. At age 10 and MAGI income of 2.79 times FPL, child1 exceeds the income limits for the regular children's group, CHIP, and every other applicable pathway, so the Medicaid category is NONE and eligibility is No."
us,scenario_046,child2_chip_eligible,1,llm_error,taxable_income_or_deductions,False,,"Child2's CHIP eligibility depends on the family's Medicaid/CHIP MAGI measured against the federal poverty line for a family of four. That MAGI is the full $92,003 of wages plus interest, which is about 280% FPL. The $14,839 FLSA overtime premium is already inside the $92,000 of wages. The 2025-2028 overtime deduction is taken below the line, in computing taxable income, so it never lowers AGI or MAGI. Once the overtime premium is subtracted from MAGI, income falls into the CHIP band. Kept in, income is above Oklahoma's CHIP ceiling for a 9-year-old, and child2 is ineligible because Medicaid is also unavailable."
us,scenario_046,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Oklahoma Medicaid eligibility requires the child to qualify through an available categorical pathway and its applicable income limit. Child 2 has no qualifying Medicaid category, receives no SSI, and the household's MAGI income of 2.79 times FPL exceeds every applicable child Medicaid threshold, so eligibility is No."
-us,scenario_046,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"To get the right answer, a model had to apply four 2026 rules together. The OBBBA qualified overtime deduction ($14,838.71 here, well under the $25,000 joint cap) is subtracted on top of the $32,200 MFJ standard deduction, giving taxable income of $44,964.29. That income is taxed under the permanent post-TCJA brackets (10% up to $24,800, 12% above), for $4,899.71. The full $2,200-per-child CTC, $4,400 in total, is then taken as a nonrefundable credit against that tax, leaving $499.71. Most wrong answers left out the overtime deduction, assumed the TCJA had sunset (personal exemptions, a 15% bracket, a $1,000 CTC), used the old $2,000 CTC, itemized by counting health insurance premiums as deductible medical expenses, or applied an AOTC even though no qualified tuition was listed."
+us,scenario_046,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"Under OBBBA rules for 2026, the household's $92,003 AGI is reduced by the $32,200 MFJ standard deduction and by the qualified overtime deduction of $14,838.71. That deduction is the FLSA overtime premium under the $25,000 joint cap. The result is $44,964.29 of taxable income. Tax is $2,480 at 10% up to $24,800 plus 12% on the remainder, or $4,899.71. The full $4,400 CTC ($2,200 per child) is used as a nonrefundable credit because it is below that liability, which leaves $499.71. Itemizing does not win: the $12,000 premium is one amount, reported under two labels; OTC costs do not count; and medical above 7.5% of AGI, mortgage interest and SALT total about $28.7k. AOTC is $0 because no qualified expenses are listed. Most wrong answers omitted the overtime deduction, applied pre-TCJA or 2025 parameters, double-counted premiums to justify itemizing, invented AOTC, or mishandled the nonrefundable CTC."
us,scenario_046,federal_refundable_credits,14,llm_error,categorical_eligibility,False,,"None of the refundable credits pays out, so federal refundable credits are $0. The EITC is fully phased out at $92,003 AGI for a two-child joint return. The American Opportunity Credit needs qualified tuition and related expenses, and none are listed, so both its nonrefundable and refundable parts are $0. The $4,400 CTC (2 x $2,200) is fully used against tax owed before credits. Even after the $32,200 standard deduction and the $14,839 overtime deduction, that tax is about $4,900, and the refundable CTC only pays back CTC left over after tax, so it is $0. The wrong answers fall into three groups: some made up tuition to claim a $1,000 refundable AOTC, some paid the $1,700-per-child refundable cap without first applying the CTC to tax, and some put tax below $4,400 so a small slice of CTC looked refundable."
us,scenario_046,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_046,payroll_tax,16,llm_error,payroll_tax_base,False,,"Employee Social Security tax is 6.2% of the head's $92,000 wages, yielding $5,704, and employee Medicare tax is 1.45%, yielding $1,334. The listed employer-sponsored insurance premium and FLSA overtime premium do not alter the $92,000 wage base here: gross wages already include overtime, and the facts do not establish a pre-tax payroll exclusion for the insurance premium. With no Additional Medicare Tax or mandatory Oklahoma employee payroll tax, total payroll tax is $7,038."
+us,scenario_046,payroll_tax,17,llm_error,payroll_tax_base,False,,"The head's $92,000 of gross wages is the FICA base. Employee Social Security is 6.2% × $92,000 = $5,704 and employee Medicare is 1.45% × $92,000 = $1,334, for a total of $7,038. There is no Additional Medicare Tax and no Oklahoma employee payroll tax. The most common mistake was subtracting the $9,208 employer-sponsored insurance premium as if it were a Section 125 pre-tax reduction, which the household facts never state, giving $6,334. The other wrong answers came from counting the overtime premium twice, inventing a state payroll tax, arithmetic or transcription slips, or giving up with $0."
us,scenario_046,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_046,state_income_tax_before_refundable_credits,45,llm_error,thresholds_rates,False,,"Oklahoma lets a household itemize only if it also itemizes on its federal return. Here the $32,200 federal standard deduction is larger than the roughly $26,309 of federal itemized deductions, so Oklahoma's $12,700 joint standard deduction applies. With four $1,000 exemptions, Oklahoma taxable income is $75,303. The 2026 Oklahoma joint rate schedule is 0% up to $7,500, 2.5% up to $9,800, 3.5% up to $14,400, and 4.5% above that, which gives $2,959.14 of tax. The nonrefundable Oklahoma child credit is 5% of the $4,400 federal CTC ($2,200 per child), or $220, leaving $2,739.14. Most models failed in one of four ways: they used the pre-2026 brackets (0.25% to 4.75%, or kept the old lower brackets with a 4.5% top rate), itemized on the Oklahoma return even though the household takes the federal standard deduction, used the federal standard deduction or the federal overtime deduction to shrink Oklahoma income, or left out or understated the 5%-of-CTC credit."
+us,scenario_046,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Oklahoma taxable income is $75,303: federal AGI of $92,003, less the fixed $12,700 joint standard deduction and four $1,000 exemptions. Oklahoma allows itemizing only when the filer itemizes federally, and the 2026 federal standard deduction of $32,200 is larger than this household's itemized deductions. The 2026 joint schedule under HB 2764 taxes the first $7,500 at 0%, $7,500–$9,800 at 2.5%, $9,800–$14,400 at 3.5%, and income above that at 4.5%, which gives $2,959.14. The nonrefundable Oklahoma child credit is 5% of the $4,400 federal CTC ($2,200 per child under OBBBA), or $220, leaving $2,739.14. Most models used the retired 0.25%–4.75% schedule, itemized or took the federal standard deduction on the Oklahoma return, cut AGI by the federal overtime deduction or ESI premiums, or left out or understated the $220 credit."
us,scenario_046,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_048,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Age 85 establishes membership in an aged population but does not itself confer Medicaid eligibility. The head receives no SSI, is not a dependent, and qualifies through none of the Medicaid pathways; the 2.17-times-FPL MAGI level also does not establish eligibility in a MAGI category. Low assets cannot substitute for satisfying a pathway's income and categorical requirements."
us,scenario_048,head_medicare_eligible,1,llm_error,other,False,,"Medicare eligibility is satisfied because the head is age 85, well above the age-65 threshold. The model’s prose reached that result but its submitted binary value reversed it: eligible must be encoded as 1, not 0."
-us,scenario_048,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies only to the $7,656 of wages: 6.2% employee Social Security tax is $474.67 and 1.45% employee Medicare tax is $111.01, totaling $585.68. Ohio adds no mandatory employee state payroll tax to this output, and the annual result is not rounded to a whole dollar."
+us,scenario_048,payroll_tax,2,llm_error,other,False,,"Payroll tax here is just employee FICA on $7,656 of wages: 6.2% Social Security ($474.67) plus 1.45% Medicare ($111.01), for $585.68. Ohio has no mandatory employee state payroll tax. Social Security survivor benefits are not covered wages. Both models arrived at about $586 in their reasoning but then put a different, unsupported number in the numeric value field. The failure is in transcribing the answer, not in the tax rule."
us,scenario_048,snap,5,llm_error,thresholds_rates,False,,"The household’s $34,626 annual income exceeds the applicable $17,420 SNAP eligibility threshold for a one-person Ohio household in 2026, so SNAP eligibility ends before any minimum-allotment calculation. A minimum benefit applies only to an otherwise eligible household; it cannot override a failed income test."
us,scenario_048,state_income_tax_before_refundable_credits,2,llm_error,thresholds_rates,False,,"The trap is Ohio's zero-tax bracket. Ohio's individual income tax schedule imposes no tax on Ohio taxable nonbusiness income at or below roughly $26,050, so this filer's $7,656 in wages — the only Ohio-taxable income, since the $26,970 in Social Security survivor benefits stays out of federal AGI at a provisional income of $21,141, below the $25,000 base amount — produces $0 of liability before any credit. Both models correctly excluded the Social Security benefits and then applied the 2.75% rate starting at the first dollar of taxable income, converting a below-threshold filer into a taxpayer. Their differing answers ($106 vs. $94.54) come only from downstream inventions — a fictitious Ohio standard deduction in one case, a recharacterized exemption credit plus the $50 senior citizen credit in the other — layered on the same missing zero bracket."
-us,scenario_049,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 OBBBA law. It starts from AGI of $246,663.72: wages net of the traditional 401(k), $3,501 interest and $6,320 dividends, minus the $721 traditional IRA contribution taken above the line. It then subtracts the $32,200 joint standard deduction, the $25,000-capped qualified overtime deduction (from the spouse's $44,647 FLSA premium) and the roughly $580 non-itemizer cash charitable deduction, giving taxable income of $188,883.95. Ordinary income is taxed through the 10/12/22% brackets and the $6,208 of qualified dividends at 15%. Most models failed for one of three reasons: they left out the $25,000 overtime deduction, they applied TCJA-sunset law (personal exemptions and 15/25/28% brackets), or they subtracted the $16,408 of employer-paid premiums from wages that never included them. The closest group disallowed the $721 IRA deduction under the active-participant phaseout, as the exclusion's corrected value ($30,702.59) does under 26 U.S.C. 219(g) and the frozen reference, which has no active-participant phase-out, does not; some also dropped the $580 non-itemizer charitable deduction or claimed auto-loan interest, which the reference does not allow. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 30,543.91; the corrected value is 30,702.59. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_049,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"The reference applies 2026 OBBBA law. It starts from AGI of $247,385, which is wages minus only the $15,436 traditional 401(k), plus $3,501 of interest and $6,320 of dividends. It then subtracts the $32,200 MFJ standard deduction, the full $25,000 qualified overtime deduction (MAGI is below the $300,000 joint phase-out start), and the roughly $580 non-itemizer cash charitable deduction, leaving $189,605 of taxable income. Tax is $29,771 on $183,397 of ordinary income under the 10/12/22% brackets (thresholds $24,800, $100,800 and $211,400), plus 15% on the $6,208 of qualified dividends. Most wrong models either left out the overtime deduction or assumed the TCJA provisions expired in 2026. Others subtracted the employer-sponsored insurance premium from the stated wages, took the auto-loan interest deduction even though it fully phases out above $200k MAGI, or missed the non-itemizer charitable deduction."
us,scenario_049,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -417,7 +419,7 @@ us,scenario_049,head_medicaid_eligible,1,parse_contract_failure,missing_output,F
us,scenario_049,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or a qualifying early-eligibility condition such as disability, end-stage renal disease, or ALS. The 53-year-old head has none of those conditions, so the eligibility result is No."
us,scenario_049,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_049,payroll_tax,31,llm_error,payroll_tax_base,False,,"The reference total of $15,134.50 has three parts. Social Security tax is 6.2% of the 2026 wage base of $184,500, which is $11,439. Medicare tax is 1.45% of the spouse's full $253,000 gross wages, which is $3,668.50. Additional Medicare Tax is 0.9% of the $3,000 above the $250,000 married-filing-jointly threshold, which is $27. Most models fell into one of two traps. About half used a stale or guessed wage base, such as the 2025 figure of $176,100 or estimates from $174,900 to $183,900. About as many subtracted the $16,408 employer-sponsored insurance premiums from the listed gross wages when computing FICA wages. That cut Medicare tax to $3,430.58 and pushed wages under the $250,000 threshold, so the Additional Medicare Tax disappeared. The prompt gives gross wages and does not say the premiums are paid pre-tax."
+us,scenario_049,payroll_tax,32,llm_error,payroll_tax_base,False,,"Only the spouse has wages: $253,000 in FICA wages, with nothing subtracted. Employee Social Security tax is 6.2% of the 2026 wage base of $184,500, or $11,439. Medicare tax is 1.45% of $253,000, or $3,668.50. Additional Medicare Tax is 0.9% of the $3,000 above the $250,000 joint threshold, or $27. Total: $15,134.50. Models missed the answer in two main ways. One group used a stale or guessed wage base (often the 2025 figure of $176,100). The other group subtracted the $16,408 employer-sponsored insurance premiums from FICA wages. That cuts Medicare tax and wrongly removes the $27 Additional Medicare Tax. A few models got $15,134.50 in their reasoning and then submitted a different number."
us,scenario_049,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -429,47 +431,43 @@ us,scenario_049,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_049,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"New Hampshire does not tax wages. Its only personal income levy, the Interest and Dividends Tax, produces no liability on this household in 2026. The trap is the rate schedule. The 2021 law cut the rate one point a year and would have left it at 1% in 2026. The 2023 law (HB 2) replaced that schedule and ended the tax for taxable periods beginning after December 31, 2024. PolicyEngine computes the same $5,021 base the models used ($9,821 of interest and dividends minus the $4,800 joint exemption) and assesses $0. Almost every wrong model found that base correctly and then applied a superseded nonzero rate: 1%, or in one case 3%."
us,scenario_049,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_051,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The reference applies the 2026 post-OBBBA single parameters: a $16,100 standard deduction against $40,000 of wages gives $23,900 of taxable income. The tax is 10% of the first $12,400 ($1,240) plus 12% of the remaining $11,500 ($1,380), for $2,620. The $180 of non-Schedule D capital gains adds no tax, because capital gain distributions are long-term gains that fall in the 0% bracket at this income. The biggest cluster (about eight models, at or near $2,641.60) taxed that $180 at 12%. Most of the rest used stale 2024/2025 parameters, assumed the TCJA had expired (bringing back personal exemptions and a 15% bracket), subtracted the $5,389 employer premium from wages, or submitted numbers that do not follow from their own arithmetic."
+us,scenario_051,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Under 2026 post-OBBBA law, a single filer with $40,000 in wages gets the $16,100 standard deduction. That leaves $23,900 of ordinary taxable income, and tax on it is 10% × $12,400 + 12% × $11,500 = $2,620. The $180 of non-Schedule-D capital gain distributions is taxed at the 0% preferential rate, because total taxable income is far below the 2026 0% LTCG threshold, so it adds nothing. The largest group of models (answering $2,641.60) taxed that $180 at the 12% ordinary rate, which adds exactly $21.60. Other models used outdated standard deductions or brackets, assumed the TCJA had expired (personal exemption and a 15% bracket), subtracted employer insurance premiums from wages, or submitted numbers that their own arithmetic did not support."
us,scenario_051,federal_refundable_credits,4,llm_error,thresholds_rates,False,,"A childless single filer with approximately $40,000 of earnings is above the 2026 EITC phaseout endpoint, so the EITC is $0. With no qualifying children and no facts supporting another refundable federal credit, all five refundable-credit components are $0."
us,scenario_051,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Louisiana Medicaid eligibility requires qualification through a specific categorical pathway, not merely acceptable immigration status or the absence of an expressly listed disqualification. This 25-year-old, nondependent adult has MAGI income of 2.51 times FPL, receives no SSI, and falls into Medicaid category NONE, so no MAGI or alternative state-program pathway confers eligibility."
us,scenario_051,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $40,000 of wages here, producing $2,480 and $580 respectively, for a $3,060 total. The dominant error was subtracting the listed employer-sponsored insurance premium without any fact establishing a pre-tax salary-reduction arrangement; several other models derived $3,060 correctly but then submitted a different number."
us,scenario_051,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_051,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Louisiana's post-2024 reform (Act 11, 2024 Third Extraordinary Session) applies a flat 3% rate to federal AGI minus a single standard deduction. That deduction was $12,500 for 2025, and R.S. 47:294(B) raises it by the CPI-U increase for calendar 2025. The December 2025 index, released January 13, 2026, puts that at 2.68% (the Legislative Fiscal Office's figure in the Louisiana Register of February 20, 2026), which gives $12,835. On $40,180 of AGI, that leaves $27,345 taxable and $820.35 of tax. Before the reference freeze the Department of Revenue applied 2.7% to the retirement income exemption, which R.S. 47:44.1 indexes in the same words (Louisiana Register, June 20, 2026); on September 28, 2026 it set the standard deduction at $12,838 the same way (RIB 26-019). That gives $820.26, within the $1 tolerance for an exact match. The $12,875 in the Department's 2026 withholding tables and estimated-tax worksheet was a provisional amount from CPI-U data available on December 1, 2025; it would give $819.15. The largest group, 21 of the 46 models, knew the flat 3% structure but used 2025's $12,500 and answered $830.40 or $830; four more reached about $830 that way and then submitted a different figure ($632 to $1,080). Of the other 21, 13 used repealed rules: 1.85%/3.5% brackets, a $4,500 exemption or deduction, or a deduction for federal income tax; six of these also subtracted the ESI premiums from AGI. The remaining eight claimed Louisiana has no income tax or that deductions offset it, gave a figure with no derivation, or gave no answer. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)."
+us,scenario_051,state_income_tax_before_refundable_credits,47,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2024 reform (Act 11) set a flat 3% rate and a $12,500 single standard deduction for 2025, and that deduction is indexed for inflation starting in 2026. The 2026 single standard deduction is $12,835, so taxable income is $40,180 − $12,835 = $27,345 and the tax is $820.35. The most common wrong answer, $830.40, used the unindexed 2025 deduction of $12,500, which leaves $10.05 too much tax. Most of the other wrong answers used pre-2025 law: the 1.85%/3.5% brackets, the $4,500 exemption, and the federal income tax deduction. Some of those also subtracted the $5,389 ESI premiums from AGI ($34,791), even though Louisiana AGI is $40,180. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
us,scenario_051,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_052,federal_income_tax_before_refundable_credits,45,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine takes the head's $43.28 traditional IRA contribution as an above-the-line deduction (AGI $504,778.56 instead of $504,821.84) without applying the IRC §219(g) active-participant phase-out (2026 MFJ range $129,000–$149,000). The head's elective 401(k) deferrals make them an active participant, and MAGI is about $504,800, so the deduction should be $0 and the value about $104,225.25.","The reference builds AGI of $504,778.56 from wages net of the 401(k) deferral ($499,073.84), $7,900 interest, $848 qualified dividends, the $3,000 capital-loss limit and a $43.28 traditional IRA deduction. It then subtracts the OBBBA 2026 MFJ standard deduction of $32,200 and taxes the $471,730.56 of ordinary income on the 2026 brackets (24% to $403,550, then 32%) for $103,865.78. To that it adds $127.20 on the qualified dividends at 15% and $218.42 of NIIT on $5,748 of net investment income. The closest models (104,225.30) match every step except the IRA deduction. Most other models went wrong for their own reasons: they assumed the TCJA sunset (pre-TCJA rates, exemptions), subtracted ESI premiums from wages, used outdated standard deductions or brackets, folded the Additional Medicare Tax into income tax, left out NIIT, or computed NIIT without netting the $3,000 capital loss. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_052,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine deducts the head's $43.28 traditional IRA contribution above the line, which is the amount beyond the $3,000 capital-loss ALD in its $3,043.28 total, without applying the IRC §219(g) active-participant phase-out. The head's 401(k) elective deferrals make them an active participant, and joint MAGI of about $505k is far above the 2026 MFJ phase-out range ($129,000–$149,000), so the statutory deduction is $0 and the statutory liability is $104,225.30, $13.89 above the reference.","The reference applies the permanent 2026 OBBBA joint-filer parameters to taxable income of $472,578.56: a $32,200 standard deduction, no personal exemptions, and a 32% bracket running from $403,550 to $512,450. The only reductions are the $926.16 401(k) deferral, the $3,000 capital-loss limit, and the $43.28 traditional IRA contribution; ESI premiums do not reduce wages and Additional Medicare Tax is excluded. The total is $103,865.78 of ordinary tax, plus $127.20 on the $848 of qualified dividends at 15%, plus $218.42 of NIIT on $5,748 (interest plus dividends, net of the $3,000 allowable loss). Most models missed because they used the expired pre-TCJA schedule with personal exemptions, subtracted the $16,208 ESI premiums from wages, used the wrong standard deduction or brackets, left out or overstated NIIT, or added the $2,250 Additional Medicare Tax. The closest models differ only because they disallowed the $43.28 IRA deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_052,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_052,head_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility requires age 65 unless a separate qualifying pathway, such as disability or end-stage renal disease, applies. The head is 59 and all unlisted disability and health-status facts are false, so no Medicare eligibility pathway applies."
-us,scenario_052,payroll_tax,31,llm_error,thresholds_rates,False,,"The correct payroll tax uses the Head's full $500,000 gross wages as the FICA base: 6.2% × the 2026 Social Security wage base of $184,500 ($11,439), plus 1.45% × $500,000 ($7,250), plus 0.9% Additional Medicare Tax on wages over the $250,000 MFJ threshold ($2,250), for $20,939. Models went wrong in three main ways. Many used a stale or guessed wage base ($168,600, $176,100, $180,000–$183,144). Several shrank the FICA base by treating the $16,208 ESI premiums or the $926 traditional 401(k) deferral as excluded from wages. Others applied the $200,000 single threshold instead of the $250,000 MFJ threshold, or submitted totals that don't match their own component calculations."
+us,scenario_052,payroll_tax,32,llm_error,payroll_tax_base,False,,"Only the head earns wages. The reference applies three components to the full $500,000: 6.2% Social Security on the 2026 wage base of $184,500 ($11,439), 1.45% Medicare on all $500,000 ($7,250), and 0.9% Additional Medicare Tax above the $250,000 married-filing-jointly threshold ($2,250), for $20,939. Two errors account for most misses. Some models used an outdated or guessed Social Security wage base, such as $168,600, $176,100 or about $180,000. Others shrank Medicare wages by subtracting the employer-sponsored insurance premium or the traditional 401(k) deferral, even though the facts never describe a cafeteria plan and 401(k) deferrals stay subject to FICA. Other models used the $200,000 single threshold, made arithmetic slips, or submitted numbers that did not match their own work."
us,scenario_052,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable disability or other qualifying pathway. The 55-year-old spouse has no listed disability or other Medicare-qualifying status, so the spouse is not eligible."
-us,scenario_053,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The reference takes the listed wages ($66,968.68) as AGI without subtracting anything, then subtracts the 2026 OBBBA single standard deduction of $16,100 to get $50,868.68 of taxable income. It then applies the permanent TCJA-structure 2026 brackets: 10% up to $12,400 ($1,240), 12% up to $50,400 ($4,560) and 22% on the remaining $468.68 ($103.11), for $5,903.11 with no nonrefundable credits. The models went wrong in five main ways: they assumed TCJA sunsets in 2026 (it does not, because OBBBA made the TCJA structure permanent); they used 2023–2025 standard deductions or brackets; they subtracted the $3,389 ESI premium from wages that were given as the taxable wage figure; they took the new-vehicle auto loan interest deduction on a $7,410 vehicle; or they submitted numbers that didn't match their own arithmetic, including invented child credits for a filer with no children."
+us,scenario_053,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The OBBBA made the TCJA structure permanent, so 2026 law uses a $16,100 single standard deduction, no personal exemption, and brackets of 10% up to $12,400, 12% up to $50,400, and 22% above that. The household's gross wages are AGI because the employer-insurance premium input does not reduce them, and the OBBBA auto-loan-interest deduction does not apply because no qualifying new-vehicle facts are given. That leaves taxable income of $50,868.68 and tax of $1,240 + $4,560 + 22% × $468.68 = $5,903.11. The models went wrong in three main ways. Some assumed the TCJA would sunset and restored the personal exemption and the 15%/25% brackets. Some used 2023–2025 or guessed 2026 amounts for the deduction and bracket thresholds. Others took $3,389 of ESI premiums (or the $950 of auto loan interest) off AGI. A few computed roughly the right figure and then submitted a different number."
us,scenario_053,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"A single 25-year-old filer with no children and $66,969 of wages receives neither a refundable Child Tax Credit nor an Earned Income Tax Credit in 2026. With the other enumerated refundable-credit components also equal to zero, federal refundable credits total $0."
us,scenario_053,payroll_tax,10,llm_error,payroll_tax_base,False,,"PolicyEngine applies the employee FICA rates to the full $66,969 of gross wages: 6.2% Social Security ($4,152.06) plus 1.45% Medicare ($971.05) equals $5,123.10. The listed employer-sponsored insurance premium ($3,389) does not reduce the payroll-tax wage base. The household facts never say the premium is paid through a Section 125 cafeteria plan, and the prompt forbids inferring unlisted facts. Eight models subtracted the premium anyway and taxed $63,580, which gives $4,863.87. The two GPT-5.4 models used the correct base and rates but multiplied them out wrong."
us,scenario_053,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_053,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"The frozen reference's Idaho taxable income is federal AGI of $66,968.68 (full gross wages; the ESI premiums do not reduce it) minus the $16,100 standard deduction, which gives $50,868.68; it leaves out Idaho's subtraction of the head's $5,000 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes on the assumption that they are not paid through a pre-tax salary reduction. Idaho's 2026 flat 5.3% rate (cut from 5.695% by 2025 HB 40) applies only above the single zero-rate bracket, which the frozen reference sets at the engine's projected $4,920: 0.053 × $45,948.68 = $2,435.28. The corrected value uses the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026: 0.053 × ($45,868.68 − $4,811) = $2,176.06; gpt-6-astra's $2,176.07 is $0.013 above it. The most common error was applying 5.3% to the whole $50,869 base, which gives $2,696.06 and ignores the zero bracket. Other models used the superseded 5.695% or 5.8% rates, subtracted the $3,389 ESI premiums from AGI, used stale or invented standard deductions ($15,000, $15,750, $8,300), or used another zero-bracket threshold: $4,489, or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)."
-us,scenario_053,state_refundable_credits,39,llm_error,state_local_rule,False,,"The whole $155 is Idaho's grocery credit (id_grocery_credit). It is a refundable credit with no income limit. This full-year resident qualifies for all 12 months with no SNAP or incarceration reduction, and the 2026 per-person base for someone under 65 is $155. The wrong answers split into two groups. The first group dropped the credit: some called it nonrefundable, some said it only offsets tax liability, some invented an income phase-out, and some never considered it. The second group used an outdated per-person amount: $100 (the pre-2023 figure), $120 (the 2023-2025 figure) or $150, instead of $155."
+us,scenario_053,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,,"Idaho taxable income here is federal AGI of $66,968.68 (wages are not reduced by the ESI premium) minus the $16,100 standard deduction, which gives $50,868.68. Idaho's 2026 schedule taxes the first $4,920 of a single filer's taxable income at 0% and everything above it at a flat 5.3%: 5.3% × $45,948.68 = $2,435.28, and no nonrefundable credits apply. Most models left out the indexed zero bracket, which pushes them to about $2,696. Others used the old 5.695%/5.8% rates or progressive brackets, used an outdated standard deduction ($15,000, $15,750 or $8,300), or subtracted the $3,389 ESI premium from AGI. The closest answers missed only because they used the 2025 or a misestimated zero-bracket amount instead of $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)."
+us,scenario_053,state_refundable_credits,40,llm_error,state_local_rule,False,,"Idaho's grocery credit is a refundable per-person income tax credit with no income limit or phase-out. For 2026, after the 2025 increase (HB 231), the amount is $155 for a resident under 65. This full-year, 25-year-old resident who received no SNAP qualifies for all 12 months, so state refundable credits equal $155. About half the models said $0, either calling the credit nonrefundable or inventing an income limit. Most of the rest knew it was refundable but used an older amount: $100, $120, or $150."
us,scenario_054,child1_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying a CHIP income ceiling alone is not sufficient. At this very low household income, the 10-year-old qualifies for Medicaid under North Carolina’s OLDER_CHILD category, and that Medicaid eligibility excludes the child from CHIP."
us,scenario_054,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Child 1 qualifies through the ordinary children's MAGI pathway: at age 10 he falls in the OLDER_CHILD category (ages 6–18), whose North Carolina income limit is 133% FPL plus the 5-percentage-point disregard, and NC folded its separate NC Health Choice tier into Medicaid so school-age children are covered under Medicaid rather than being pushed to CHIP. The household's only income is $17,100 of self-employment earnings, which after the deductible half of SE tax leaves MAGI at 0.58 × FPL for a family of three — a fraction of the applicable limit. Both models returned 0 without ever performing that categorical-plus-income test: one asserted an income failure that the arithmetic contradicts, the other treated unlisted household facts as missing prerequisites even though the prompt instructs that unlisted booleans are false and take-up is assumed."
us,scenario_054,child1_wic_eligible,2,llm_error,age_disability,False,,"WIC categorical eligibility covers pregnant, postpartum, and breastfeeding women, infants, and children under age five; it does not extend to nutritionally at-risk school-age children. Child 1 is age 10, so income eligibility and another household member’s WIC receipt cannot make Child 1 eligible."
us,scenario_054,child2_chip_eligible,4,llm_error,categorical_eligibility,False,,"The decisive CHIP rule is categorical: a child who qualifies for Medicaid cannot also qualify for CHIP. Child 2 qualifies for Medicaid in the young-child category, so the CHIP income ceiling and employer-sponsored insurance do not alter the result."
us,scenario_054,child2_early_head_start_eligible,2,llm_error,categorical_eligibility,False,,"Early Head Start serves children from birth to age 3 in households at or below 100% of the federal poverty line, and Child 2 at age 2 clears the age band while the household clears the income test. The income trap is that the only earnings listed, $17,100 of ""sstb self employment income before lsr,"" is a QBI-classification input that does not feed self-employment income, leaving countable household income at $0 — and even taken at face value, $17,100 is below 100% FPL for a three-person household. Both wrong answers denied eligibility: one by treating that phantom income plus a nonexistent asset test as disqualifying, the other by reversing the under-3 age comparison."
-us,scenario_054,child2_head_start_eligible,3,llm_error,categorical_eligibility,False,,"Head Start for preschool-age children requires the child to be at least age 3, while Early Head Start serves children under age 3. Child 2 is age 2 and therefore fails the categorical age requirement regardless of household income."
+us,scenario_054,child2_head_start_eligible,4,llm_error,age_disability,False,,"Preschool Head Start serves children from age 3 until they reach kindergarten age. Infants and toddlers under 3 fall under Early Head Start. Child 2 is 2, so the age test fails no matter how low the household income is. Three models treated the low income as enough or wrongly put age 2 inside the preschool range. The fourth model applied the age cutoff correctly but still submitted a value of 1."
us,scenario_054,child2_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"A 2-year-old qualifies for Medicaid through the age-based YOUNG_CHILD MAGI category, whose North Carolina income standard for ages 1-5 exceeds 200% FPL; this household sits at 0.58 x FPL, so the income test is satisfied outright. The trap is that Medicaid, unlike CHIP, imposes no bar on children who already hold employer-sponsored insurance, and it requires no disability, SSI, or other special categorical status beyond age. All three wrong answers refused the category: one swapped in CHIP's other-coverage exclusion, one measured the child against an adult-level FPL cutoff, and one defaulted to 0 for want of an extra eligibility flag that the children's pathway never requires."
us,scenario_054,child2_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC eligibility for a child turns on two things only: falling in the child category (age 1 through the fifth birthday) and household income at or below 185% of the federal poverty guideline, with adjunctive eligibility through Medicaid, SNAP, or TANF as an independent qualifying path. This three-person North Carolina household has at most $17,100 of self-employment income against a 185% FPL limit near $49,000 for 2026, so the income screen is cleared with enormous margin, and the head's listed WIC receipt reflects a household already certified under that same screen. Both wrong answers denied a child who satisfies the age category and the income test — one by inventing an income ceiling the household never approaches, the other by treating an unlisted enrollment flag as dispositive of eligibility."
us,scenario_054,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_054,federal_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"For a self-employed filer, earned income for both the EITC and the refundable CTC is net SE profit minus the deductible half of SE tax. Here that is $17,100 - $1,208.07 = $15,891.93. The EITC is still in its 40% two-child phase-in, giving $6,356.77. The 2026 refundable CTC uses the permanent OBBBA $2,500 earned-income threshold, giving 15% x $13,391.93 = $2,008.79, which is under the $1,700-per-child refundable cap. The wrong answers split into four groups. Some used the wrong earned-income base: gross $17,100, the 92.35% figure of $15,791.85, or a double-reduced $14,584. Some used the pre-TCJA $3,000 threshold or the $1,000-per-child CTC. Some dropped a component or misplaced the EITC on its schedule. The rest denied eligibility outright."
+us,scenario_054,federal_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"For a self-employed filer, earned income for both the EITC and the ACTC is SE profit minus the deduction for half of SE tax: $17,100 − ($17,100 × 0.9235 × 0.153 / 2 = $1,208.07) = $15,891.93. That amount is still in the two-child EITC phase-in, so the EITC is 40% × $15,891.93 = $6,356.77. Because regular tax is $0, the refundable CTC is 15% × ($15,891.93 − $2,500) = $2,008.79. The $2,500 floor was made permanent by OBBBA, and the result is below the $1,700-per-child cap. Most models went wrong in one of three ways: they used the wrong earned-income base (gross $17,100, the 92.35% net-earnings figure of $15,791.85, or a double-reduced $14,584); they used the expired $3,000 ACTC threshold or the $1,000-per-child CTC; or they dropped one of the two credits entirely."
us,scenario_054,free_school_meals_eligible,2,llm_error,thresholds_rates,False,,"The decisive step is the free-meal income test: $17,100 of self-employment income for a household of three is 63% of the federal poverty guideline, less than half the 130% ceiling for free meals, and the household is independently categorically eligible through SNAP ($524.80/month), which by itself confers free-meal status for the 10-year-old K-12 student. NC has no universal free school meal program, so the household qualifies through the standard free tier rather than a statewide waiver, and PolicyEngine values the benefit at $7.15/day for one enrolled child ($1,130.96/year). Both wrong models denied eligibility for a household at less than one-third of the free-meal income limit — one by asserting the income exceeded the threshold, the other by treating the absence of an explicit eligibility flag in the prompt as a negative."
us,scenario_054,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The head is a 40-year-old living with two of their own children, which places them in the parent/caretaker-relative Medicaid category (medicaid_category = PARENT), a mandatory MAGI pathway that exists in every state independent of expansion status. Their only income is $17,100 of self-employment earnings, which after the deductible half of SE tax gives MAGI at 0.58 x FPL for a household of three — far below both the parent/caretaker limit and the 138% FPL adult limit North Carolina adopted when it implemented expansion on December 1, 2023 (S.L. 2023-7), in force for 2026. Every wrong model either declared the head a childless/non-categorical adult despite two listed children, or asserted NC is still a non-expansion state, and so answered 0 on a household that clears the income test by a wide margin on two independent pathways."
-us,scenario_054,head_wic_eligible,16,llm_error,categorical_eligibility,False,,"WIC eligibility is decided person by person, and each person must first fall into a categorical group. An adult qualifies only if pregnant, postpartum or breastfeeding, and a child qualifies only if under age 5. The 40-year-old head has none of the pregnancy, postpartum or breastfeeding flags, and unlisted status inputs count as false, so her WIC category is none and income never enters the test. The trap is the ""receives wic"" flag listed under the head. It reports that the household takes up WIC, which here happens through the 2-year-old, and it gives no adult categorical status. Every model treated either that receipt or being the caretaker of a child under 5 as making the head eligible herself."
+us,scenario_054,head_wic_eligible,17,llm_error,categorical_eligibility,False,,"WIC eligibility depends on a person's own category: pregnant, postpartum (up to 6 months after delivery), breastfeeding (up to 12 months), infant, or child under 5. Low income alone doesn't qualify anyone. The head is a 40-year-old adult with no pregnancy flag, and the youngest child is 2, so the postpartum and breastfeeding windows ended long ago. The head therefore fits no WIC category, and only Child 2 is eligible. The 'receives wic' fact is a receipt/take-up input and does not make anyone eligible. Every model treated that receipt flag, or the head's role as parent of a child under 5, as a pathway to eligibility."
us,scenario_054,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages, not SSTB self-employment income; that income instead enters the separate self-employment-tax calculation. With no reported wages and all unlisted numeric inputs set to zero, every component of the requested employee-side payroll tax is zero."
us,scenario_054,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. The household is in the FREE tier because it meets categorical eligibility through SNAP or TANF and its school-meal income is 63% of the federal poverty guideline, below the 130% free-meals threshold; therefore reduced-price support is zero."
-us,scenario_054,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax equals 15.3% of the statutory net-earnings base: $17,100 × 92.35% × 15.3% = $2,416.15305, which rounds to $2,416.15. The deduction for one-half of self-employment tax affects adjusted gross income, not the self-employment tax liability itself, and the requested amount requires cent-level arithmetic rather than whole-dollar estimation."
-us,scenario_054,snap,44,llm_error,thresholds_rates,False,,"The correct monthly allotment is the FY2026 three-person maximum of $785 minus an expected contribution of 30% of $931 net income ($1,425 gross minus the $285 earned income deduction and the $209 standard deduction). That contribution rounds up to a whole $280 under 7 CFR 273.10(e)(2)(ii)(A), which gives $505 a month and $6,060 a year. North Carolina's broad-based categorical eligibility means the $14,000 bank balance does not disqualify the household. The wrong answers fall into four groups:
-- Right inputs, no round-up: nine models used $279.30 instead of $280 and landed at $6,068–$6,072.
-- Wrong parameters: most used stale or misremembered ones, such as the FY2024 $766/$198 or FY2025 $768/$204 pairs, or maximum allotments of $800 or more.
-- Asset test: six zeroed the benefit with a resource test that categorical eligibility waives.
-- Income or arithmetic: the rest miscounted self-employment income or made arithmetic errors."
+us,scenario_054,self_employment_tax,7,llm_error,other,False,,"This is a plain SECA calculation. Net SE earnings are $17,100 × 0.9235 = $15,791.85, which is far below the Social Security wage base, so the full 15.3% applies: $15,791.85 × 0.153 = $2,416.15. Every model either picked the right method and then made an arithmetic or rounding slip, or worked out about $2,416 and then submitted some other number. Two models subtracted things that do not reduce SE tax at all, such as the deduction for half of SE tax, which only lowers income-tax AGI."
+us,scenario_054,snap,45,llm_error,thresholds_rates,False,,"The household has $1,425/month of gross self-employment earnings. Its net income is $931 after the $285 earned income deduction (20%) and the FY2026 $209 standard deduction for three people. The expected contribution is 30% of $931, rounded up to $280, and the $785 three-person maximum minus $280 gives $505/month, or $6,060 a year. The models split into four groups of errors: (1) models that used the right inputs but did not round the contribution up, getting $505.70 ($6,068.40) or $506 ($6,072); (2) models that used stale or invented maximum allotments and standard deductions from FY2024, FY2025, or rough projections; (3) models that wrongly reduced self-employment income by SE tax or by a 40% cost allowance; (4) models that zeroed out the benefit by applying a resource limit, which North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives."
us,scenario_054,state_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Two independent paths lead to $0 of North Carolina tax. First, the only income fact listed is the SSTB *component* input; total self-employment income is unlisted and therefore 0 under the prompt's ""treat any unlisted numeric input as 0"" rule, so the engine's trace shows essentially no income entering nc_income_tax. Second, even crediting the full $17,100 as net self-employment earnings, federal AGI is $17,100 − $1,208 (half of SE tax) = $15,892, and North Carolina's 2026 head-of-household standard deduction of $19,125 — the head has two qualifying children — exceeds that, driving NC taxable income to $0 and the 3.99% 2026 flat rate onto a zero base. Every wrong model produced a positive liability; two of them derived exactly $0 in their own reasoning and then submitted a different number anyway."
-us,scenario_055,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The reference subtracts three amounts from AGI of $156,275.86, totaling $20,273.45: the 2026 standard deduction of $18,150 ($16,100 plus the $2,050 aged add-on), the OBBBA senior deduction of $6,000 phased down by 6% of MAGI above $75,000 to $1,123.45, and the new $1,000 non-itemizer cash charitable deduction (IRC 170(p), in effect from 2026), which applies because the filer takes the standard deduction and gave $15,647 in cash. That leaves taxable income of $136,002.41. The 2026 brackets, made permanent under OBBBA, tax it at $17,966 through $105,700 plus 24% of the rest. The closest models (claude-fable-5, claude-sonnet-5.5, gpt-6-luna) got every step except the $1,000 non-itemizer charitable deduction and came out exactly $240 high. Eleven models applied the pre-TCJA sunset rules, with personal exemptions and 15/25/28% brackets. Most of the rest dropped or mis-phased the senior deduction or used guessed standard deductions and brackets."
+us,scenario_055,federal_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"The 2026 result follows OBBBA law. AGI is $156,275.86, with 85% of Social Security taxable. The filer takes the $18,150 standard deduction ($16,100 base plus $2,050 aged add-on). The $6,000 senior deduction phases down by 6% of MAGI above $75,000, leaving $1,123.45. The new §170(p) charitable deduction for non-itemizers adds $1,000 for a single filer. That leaves taxable income of $136,002.41, taxed at the OBBBA-reset brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%). The closest models handled everything except the $1,000 non-itemizer charitable deduction and came out exactly $240 high. A large group applied TCJA-sunset law (personal exemption, 15/25/28% brackets), which OBBBA repealed. Others left out the senior deduction, did not phase it out, or used inflation-projected parameters."
us,scenario_055,payroll_tax,5,llm_error,thresholds_rates,False,,"The only FICA-taxable income here is the $83,635 of wages: the $67,760 taxable pension and $5,742 Social Security retirement benefit are outside the payroll tax base, the wage is far below the 2026 Social Security wage base, age 70 does not exempt wages from FICA, and Florida levies no mandatory employee state payroll tax. The answer is therefore exactly 6.2% × $83,635 = $5,185.37 plus 1.45% × $83,635 = $1,212.71, or $6,398.08. What separates the reference from the wrong answers is execution of that single multiplication at the employee-side rate: two models stated the right rule and then mis-multiplied or rounded, one submitted a number contradicting its own correct explanation, one used the combined 12.4%/2.9% employer-plus-employee rates before guessing, and one submitted a placeholder zero."
us,scenario_056,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The HSA deduction and other applicable adjustments reduce adjusted gross income to -$5,882.05, and taxable income remains zero after the standard deduction. With zero income tax before credits and no other included tax components, federal income tax before refundable credits is $0."
us,scenario_056,head_medicaid_eligible,12,llm_error,health_coverage,False,,"New Jersey’s ACA expansion covers this 20-year-old as a MAGI-based adult because the person satisfies the immigration rule and the engine’s MAGI income level is 0.00 times FPL. Employer-sponsored insurance does not disqualify an otherwise eligible adult from Medicaid; it affects other health-coverage calculations, including ACA premium assistance. The wrong answers either imposed that nonexistent ESI exclusion or replaced the Medicaid MAGI calculation with a sum of inputs that the engine does not count as MAGI in that manner."
@@ -477,25 +475,25 @@ us,scenario_056,head_wic_eligible,1,parse_contract_failure,missing_output,False,
us,scenario_056,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_056,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage earnings, and the household has no reported wages or salary. The listed SSTB self-employment income belongs in the separately requested self-employment-tax calculation, while alimony and financial assistance are not employee FICA wages."
us,scenario_056,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax applies because $4,340 exceeds the $400 filing threshold. Multiplying the 92.35% net-earnings base by the combined 15.3% Social Security and Medicare rate yields $613.22; the threshold does not reduce the taxable base or eliminate liability."
-us,scenario_056,snap,45,prompt_ambiguity,categorical_eligibility,False,,"Countable SNAP gross income is about $1,486.54 a month: $361.67 of self-employment income, $124.92 of alimony and $1,000 of financial assistance. That is 112% of FPL, well under NJ's broad-based categorical eligibility (BBCE) gross limit, and the asset test is waived. After the $213.68 standard deduction and the $72.33 earned-income deduction, adjusted income is $1,200.53. Shelter costs are only the $995.75 in monthly real-estate taxes; the tax-side home-mortgage-interest input is not a SNAP shelter cost. That gives an uncapped excess shelter deduction of $395.49 and net income of $805.03. The formula benefit is $304.68 − $241.50 = $63.18, and New Jersey's $95 state minimum benefit raises it to $95 a month ($1,140 a year). Models either added mortgage interest and hit the shelter cap (about $1,900), left out the shelter deduction or the NJ minimum (and gave $0), or dropped the financial assistance (and gave the maximum allotment). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)."
+us,scenario_056,snap,46,prompt_ambiguity,categorical_eligibility,False,,"Gross income counts the $12,000 financial assistance as unearned income, giving $1,486.54/month: $361.67 self-employment, $124.92 alimony and $1,000 assistance. That is 112% of FPL, well under NJ's broad-based categorical eligibility limit. After the $213.68 standard deduction and the $72.33 earned-income deduction, adjusted income is $1,200.53. Shelter costs are only the $995.75/month real estate taxes. Home mortgage interest is a tax-deduction input and is not a SNAP shelter cost here, so the excess shelter deduction is an uncapped $395.49 and net income is $805.04. The formula benefit is $304.68 − $241.50 = $63.18, which New Jersey's $95 state minimum benefit replaces, giving $1,140 a year. Models went wrong in four ways: they excluded the assistance, counted mortgage interest and hit the shelter cap, declared the household ineligible, or zeroed out a sub-maximum formula without applying the NJ $95 floor. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)."
us,scenario_056,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_056,state_income_tax_before_refundable_credits,5,llm_error,thresholds_rates,False,,"New Jersey's gross income tax exempts a single filer entirely when New Jersey gross income falls at or below the $10,000 minimum filing threshold, and this household's NJ gross income is $5,838.53 (alimony $1,498.53 plus self-employment $4,340.00; the $12,000 of financial assistance is not NJ gross income). PolicyEngine therefore computes the bracket tax as an intermediate figure — $4,838.53 of taxable income after the $1,000 regular exemption, times the 1.4% first-bracket rate, equals $67.74 — and then zeroes it out under the below-threshold exemption, leaving $0.00. Every wrong model stopped at the 1.4% bracket arithmetic and never applied the threshold exemption; three of them also mis-specified the NJ income and deduction base along the way."
-us,scenario_056,state_refundable_credits,46,llm_error,credit_phaseout,False,"The $265.60 reference is exactly 40% × $664, the full 2026 childless EITC maximum. The derivation itself says the credit phased in to only $308.55 (7.65% × $4,033.39), and no phase-out applies with negative AGI and $4,033 of earnings. Under N.J.S.A. 54A:4-7 and IRC §32(a), the credit should be 40% × $308.55 = $123.42, so PolicyEngine's NJ path for young childless filers appears to use eitc_maximum (minus the reduction) instead of the phased-in amount.","The key rule is that New Jersey pays its 40% EITC match to childless filers aged 18 and over who fail the federal 25–64 age test. Most models stopped at 'federal EITC = $0, so NJ EITC = $0' or used the outdated 21+ age floor. The models that found the age-18 pathway computed 40% of the phased-in federal-formula credit (7.65% × $4,033.39 = $308.55), giving $123.42. The reference value of $265.60 equals 40% of the full $664 childless maximum, so it skips the earned-income phase-in. Two models also used gross self-employment income without subtracting half of the self-employment tax, and two others claimed the $50 property tax credit, which is refundable only for filers who are 65 or older, blind, or disabled. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)."
+us,scenario_056,state_refundable_credits,47,llm_error,credit_phaseout,False,"PolicyEngine's NJ EITC formula for childless filers aged 18 to 24 caps the unscaled $308.55 federal-formula phase-in amount at 40% of the $664 2026 federal maximum ($265.60). N.J.S.A. 54A:4-7 instead requires 40% of the full federal-formula credit, which gives 0.40 × $308.55 = $123.42.","The deciding rule is that New Jersey gives its refundable EITC to filers aged 18 and over who have no qualifying children. It is computed with the federal formula even though the federal childless credit requires age 25. Most models tied NJ eligibility to federal eligibility or to an outdated age-21 floor, and returned $0 or a $50 property-tax credit. EITC earned income here is $4,340 minus half of SE tax ($306.61), which is $4,033.39, so the phase-in amount is $308.55. Models that applied NJ's 40% match to that amount got $123.42. The reference's $265.60 equals 40% of the $664 2026 maximum childless credit, which the engine reaches by capping the $308.55 phase-in amount at that scaled maximum. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)."
us,scenario_056,tanf,2,llm_error,categorical_eligibility,False,,"TANF is categorically limited to assistance units containing a dependent child: New Jersey's WorkFirst NJ/TANF requires a needy child under 18 (or 18 and a full-time secondary student) in the unit, and childless adults are served instead by WFNJ General Assistance, which is not TANF. This household is a single 20-year-old head with no children, so the benefit is $0 regardless of the income test. The second trap is that `financial_assistance` is an input income line in the household facts, not a computed program output; echoing it back as the TANF benefit both invents categorical eligibility and confuses an input with the requested computed value."
-us,scenario_057,federal_refundable_credits,35,llm_error,thresholds_rates,False,,"The household's only refundable credit is the no-qualifying-child EITC. On a joint return, the age test is met when either spouse is 25-64, and the head is 43. EITC earned income is about $9,665: $10,400 of SE profit minus the deductible half of SE tax. That is past the 2026 phase-in end (about $8,700) and well below the joint phase-out start (about $18,000), so the credit sits on the plateau at the 2026 statutory maximum of $664. The wrong answers fall into three groups. Some used the wrong maximum: the 2025 figure of $649, or guessed inflation-indexed values from $620 to $667. Some misplaced the household in the phase-in or phase-out range. The rest zeroed the credit through invented disqualifiers: requiring both spouses to be 25 or older, requiring wages, requiring positive tax liability, or claiming an income or disability-benefit bar."
-us,scenario_057,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The head is blind and disabled with only $10,400 of self-employment income, which after the SSI $20 general, $65 earned-income, and one-half-of-remainder exclusions leaves countable income far below the federal benefit rate, producing $7,231 of SSI for the year. Louisiana is a Section 1634 state, so SSI receipt is itself a Medicaid eligibility category (medicaid_category = SSI_RECIPIENT) conferring automatic coverage with no income test applied. The trap is that all three models tested income or hunted for a MAGI category instead of following the SSI-receipt pathway, and each also ignored that Louisiana's ACA adult expansion covers 19-64-year-olds to 138% FPL while this household's MAGI is 0.45 x FPL — two independent routes to Yes."
+us,scenario_057,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household's only refundable credit is the no-qualifying-child EITC, and it gets the full 2026 maximum of $664. The joint return passes the age test because one spouse, the head at 43, is between 25 and 64; only one spouse has to qualify. Earned income is about $9,665: $10,400 of self-employment income minus the deductible half of self-employment tax. That is above the 2026 phase-in ceiling of about $8,680 and well below the married-filing-jointly phase-out start of about $18,000, so the credit sits on the plateau. Models split two ways. Some denied the credit outright, citing the age test, treating self-employment income as non-earned, or claiming the household was in the phase-out. The rest found the plateau but used a stale or mis-indexed maximum ($649 for 2025, or $666–$667) or made arithmetic and phase-out errors."
+us,scenario_057,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"The head is blind and disabled, with very low countable income ($10,400 of self-employment income, $14 of disability benefits) and only $912 in assets. That makes the head an SSI recipient, receiving about $7,231 a year. Louisiana gives automatic Medicaid to SSI recipients, so the engine puts the head in the SSI_RECIPIENT category. The head would also qualify under Louisiana's adult expansion, because household MAGI is only 0.45× FPL, far below the 138% limit. All four models missed the SSI pathway, and none of them checked the head's income against the expansion limit correctly."
us,scenario_057,head_medicare_eligible,10,llm_error,age_disability,False,,"Before 65, a person qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or through the ESRD/ALS pathways. PolicyEngine checks this with months of Social Security disability receipt, and it does not use the is_disabled or is_blind flags. The head is 43. Their only disability income is $14 of generic ""disability benefits"", which is not Social Security disability. No months of SSDI receipt are listed, so that input is 0, and neither ESRD nor ALS is listed either. Every model treated disability status plus a nominal disability-benefit amount as enough for Medicare under 65. That skips both the SSDI-entitlement requirement and the 24-month waiting period."
us,scenario_057,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_057,self_employment_tax,7,llm_error,thresholds_rates,False,,"The whole case reduces to one canonical computation: net earnings from self-employment are 92.35% of the $10,400 SSTB profit ($9,604.40), and the combined OASDI/HI rate of 15.3% applies in full because $9,604.40 is far under the 2026 Social Security wage base, giving 0.153 × 9,604.40 = $1,469.47. The $400 figure in §1402(b) is the floor at which SE tax begins, not a ceiling below which it is waived, and no other floor exists; the one-half-of-SE-tax figure is an above-the-line deduction under §164(f), not the liability. The prompt further defines this output as SE tax excluding Additional Medicare Tax, and the 0.9% AMT threshold for joint filers is $250,000 of combined earnings, so nothing is added on top. Every wrong answer comes from inverting the $400 floor, inventing a different floor, truncating the 0.9235 base factor, reporting the deduction instead of the tax, or bolting on a nonexistent AMT."
-us,scenario_057,snap,46,prompt_ambiguity,age_disability,False,,"The trap is SNAP countable income. The household's SNAP gross income has to include the head's computed SSI payment of $602.58/month. That is the individual rate: the $994 FBR minus $391.42 of countable income from the head's own $866.67 of self-employment earnings and $1.17 of benefits. Adding it to $866.67 of earnings and $3.17 of disability benefits gives gross income of $1,472.42. After the 20% earned-income deduction and the $209 standard deduction, net income is $1,090.08. The benefit is $546 − $327 = $219/month for January–September and $232.74/month for October–December under FY2027 parameters, totaling $2,669.22. About 18 models left SSI out entirely and got net income near $488 and roughly $4,700/year. About 17 counted SSI at the couple rate of about $1,099/month and got net income near $1,586 and roughly $840/year. Most of the rest wrongly zeroed out the benefit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
+us,scenario_057,snap,47,prompt_ambiguity,age_disability,False,,"The trap is the SSI amount, which must be counted as unearned income for SNAP. The engine gives the head $602.58/month of SSI: the individual rate of $994 minus $391.42 of countable income ($866.67 of self-employment income, minus the leftover $18.83 general exclusion and the $65 earned-income exclusion, then halved). Monthly SNAP gross income is therefore $866.67 + $3.17 + $602.58 = $1,472.42. Subtracting the 20% earned-income deduction ($173.33) and the $209 standard deduction leaves net income of $1,090.08, so the benefit is $546 − $327 = $219/month for January–September. October–December use FY2027 parameters ($558.24 − $325.50 = $232.74/month), for a total of $2,669.22. Roughly half the models left SSI out of SNAP income and got about $4,700. Most of the others counted SSI at the couple rate of about $1,099/month and got about $840. The rest called the household ineligible or produced unsupported numbers. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
us,scenario_057,spouse_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires the person to be outside Medicaid eligibility; a low income alone does not establish CHIP eligibility. The spouse is Medicaid-eligible in Louisiana's adult category, so the Medicaid exclusion makes `spouse_chip_eligible` false without applying a CHIP income threshold."
us,scenario_057,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Louisiana adopted the ACA adult expansion effective July 1, 2016, so any non-elderly adult aged 19–64 who is not otherwise eligible qualifies for Medicaid on income alone at or below 138% FPL under MAGI methodology — no pregnancy, parenthood, or disability category is required. The spouse is 20 with household MAGI at 0.45 x FPL (the only income is the head's $10,400 SSTB self-employment plus $38 of disability benefits, against a two-person 2026 FPL near $21,150 and a 138% cutoff near $29,200), placing the household roughly a third of the way to the expansion limit. Every wrong model either invented an income excess that does not exist or applied pre-ACA categorical gatekeeping that Louisiana abolished, and each therefore returned 0 where the ADULT expansion pathway returns eligible."
us,scenario_057,spouse_medicare_eligible,8,llm_error,age_disability,False,,"To be eligible for Medicare before age 65, a person must be entitled to Social Security Disability Insurance (SSDI) and have received it for 24 months, or have ESRD or ALS. Being disabled is not enough on its own. The spouse is 20 and has only $24 of generic `disability_benefits`. That is private or other disability income, not SSDI. Months of SSDI receipt and ESRD/ALS status are not listed, so they default to 0/false. The spouse fails both the age-65 test and the disability pathway. Every model read ""is disabled"" plus a small disability-benefit amount as satisfying the disability pathway."
us,scenario_057,spouse_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC does not cover women merely because they are young, disabled, or income-eligible. A woman must be pregnant, breastfeeding, or postpartum; the spouse has none of these statuses, and the household has no infants or children under age five."
-us,scenario_057,ssi,46,prompt_ambiguity,age_disability,False,,"The decisive step is that the 20-year-old spouse does not meet the SSI aged/blind/disabled test. That makes the head an eligible individual with an ineligible spouse, not half of an eligible couple. The head is therefore paid from the $994/month individual FBR, not the $1,491/month couple rate. None of the spouse's $2/month disability income is deemed because it is below the $497 couple-minus-individual threshold. The head's $1.17/month of unearned income uses part of the $20 general exclusion, and the remaining $18.83 applies to earnings: ($866.67 − $18.83 − $65) / 2 = $391.42 countable income. SSI is $994 − $391.42 = $602.58/month, or $7,231/year. Almost every nonzero answer used the eligible-couple rate and pooled income, producing about $12,600–$13,600. The $0 answers wrongly concluded that income or assets exceeded the SSI limits, or never calculated the benefit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
+us,scenario_057,ssi,47,prompt_ambiguity,age_disability,False,,"Under PolicyEngine's rules, only the head passes the SSI aged/blind/disabled test. The spouse has is_ssi_aged_blind_disabled=False, so the head is paid the individual federal benefit rate of $994/month as an eligible person with an ineligible spouse, not the $1,491 couple rate. The spouse's $2/month of unearned income is below the $497 deeming threshold (the couple rate minus the individual rate), so nothing is deemed. The head's own $1.17/month of unearned income uses up part of the $20 general exclusion. The remaining $18.83 and the $65 earned-income exclusion come off $866.67 of monthly earnings, and half of what is left is countable: $391.42. The benefit is $994 − $391.42 = $602.58/month, or $7,231/year. Most models paid an eligible-couple benefit of about $13,100. The $0 answers either skipped the one-half earned-income exclusion or wrongly treated earnings, assets, or the absence of listed SSI receipt as disqualifying. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
us,scenario_057,state_income_tax_before_refundable_credits,3,llm_error,state_local_rule,False,,"The trap is Louisiana's flat-tax reform, effective tax year 2025: Act 11 of the 2024 Third Extraordinary Session replaced the graduated 1.85/3.5/4.25% schedule and the $4,500-per-filer personal exemptions with a single 3% rate and a $25,000 standard deduction for married filing jointly (indexed from 2026), and Louisiana taxable income starts from federal AGI. This household's only taxable income is $10,400 of SSTB self-employment income, yielding federal AGI of about $9,665 after the deductible half of self-employment tax — far below the joint standard deduction — so Louisiana taxable income and tax are both zero. Both wrong answers subtracted $9,000, the repealed exemption figure, manufacturing a $665 taxable base that the current deduction fully absorbs."
-us,scenario_057,state_refundable_credits,34,llm_error,state_local_rule,False,,"This is a married couple with no children. The head is 43, and his net self-employment earnings after the half-SE-tax deduction are about $9,665 ($10,400 minus about $735). That is above the roughly $8,680 point where the childless phase-in ends and well below the MFJ phase-out start, so the federal childless EITC is at its 2026 maximum of $664. Louisiana's refundable EITC is 5% of the federal EITC, which gives $33.20. Most models answered $0 for one of three reasons: they did not know Louisiana has a refundable EITC, they assumed the EITC requires children or that the 20-year-old spouse disqualifies the couple, or they put federal EITC at zero. The models that did apply the Louisiana EITC used a stale federal maximum ($632, $649 or $630), the wrong Louisiana match rate (3.5% or 10%), or a federal EITC that was never capped at the maximum."
-us,scenario_059,federal_income_tax_before_refundable_credits,12,llm_error,taxable_income_or_deductions,False,,"Provisional income is $25,600 + $18,552 = $44,152, only $152 above the $44,000 MFJ adjusted base. Taxable Social Security is therefore $6,000 + 85% x $152 = $6,129.20, and AGI is $31,729.20. Under 2026 law (TCJA made permanent by OBBBA), deductions total $47,500: the $32,200 MFJ standard deduction, $1,650 x 2 additional aged amounts, and the $6,000-per-person senior deduction (no phase-out at this MAGI). That is far above AGI, so taxable income and tax are $0. Most models got AGI right but used TCJA-sunset deductions and personal exemptions of about $28-30K and left out the senior deduction. The rest over-taxed Social Security with a flat 85% or a mangled worksheet and also left out the $12,000 senior deduction."
+us,scenario_057,state_refundable_credits,35,llm_error,state_local_rule,False,,"This joint-filing couple with no children qualifies for the federal childless EITC. The head is 43, and on a joint return only one spouse has to be 25–64. The household's earned income is about $9,665 ($10,400 of self-employment income minus half of the roughly $1,469 SE tax). That is above the childless phase-in ceiling and below the joint-return phase-out start, so they get the full 2026 maximum of $664. Louisiana's refundable EITC is 5% of the federal EITC, or $33.20. Most models wrongly said Louisiana has no refundable EITC, or no income tax at all, or set the federal EITC to zero. The rest used the wrong year's federal maximum or the wrong state match rate."
+us,scenario_059,federal_income_tax_before_refundable_credits,13,llm_error,taxable_income_or_deductions,False,,"Provisional income is $25,600 + $18,552 = $44,152, only $152 above the $44,000 MFJ second-tier threshold. Taxable Social Security is therefore $6,000 + 0.85 × $152 = $6,129.20, which makes AGI $31,729.20. For 2026, OBBBA made the TCJA standard deduction permanent: $32,200 for MFJ plus $1,650 for each spouse aged 65+, for a total of $35,500. Each spouse also gets the new $6,000 senior deduction, adding $12,000. Total deductions of about $47,500 are far above AGI, so taxable income and tax are $0. Most models went wrong in one of two ways: they assumed the TCJA would sunset and used the pre-TCJA standard deduction plus personal exemptions (about $29–30.5k), or they overstated taxable Social Security by applying 85% to all benefits instead of using the tiered formula."
us,scenario_059,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered earnings, not Social Security retirement benefits or taxable IRA distributions. With no wages or self-employment income listed, the employee payroll-tax base is zero and the resulting payroll tax is $0."
us,scenario_060,federal_refundable_credits,1,llm_error,age_disability,False,,"For 2026, the temporary 2021 removal of the upper age limit for the childless EITC has expired. A taxpayer without a qualifying child must be under age 65, so this 70-year-old receives no EITC and all other refundable credit components are also zero."
us,scenario_060,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Texas Medicaid does not grant eligibility merely because a 70-year-old has MAGI of 1.08 times FPL or minimal bank assets. The head must qualify through a specific Medicaid category, and the aged pathway is not satisfied; the engine therefore assigns Medicaid category NONE."
@@ -514,14 +512,14 @@ us,scenario_064,child1_medicaid_eligible,9,llm_error,thresholds_rates,False,,"Th
us,scenario_064,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,child1_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC eligibility requires both a categorical status and an income test, and this household fails both. Categorically, the ""child"" category covers children from their first birthday only until their fifth birthday; Child 1 is 12 and falls outside every WIC category (pregnant, postpartum, breastfeeding, infant, child under 5). Independently, the income test caps countable household income at 185% of the federal poverty guideline, and this household's roughly $98,810 of income for a 5-person unit is far above that ceiling. Both gates independently force is_wic_eligible = False."
us,scenario_064,dependent1_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,dependent1_medicaid_eligible,30,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent1 is a 27-year-old with no income of their own: 20 usual weekly hours but $0 of wages. The engine puts dependent1's MAGI income level at 0.00 x FPL and places them in the MAGI-based ACA adult category, which returns eligible. That category has no asset test (42 U.S.C. 1396a(e)(14)(C)), and dependent1's employer-sponsored coverage and disability status do not block it. Every answering model instead charged dependent1 with the parents' income (the head's ~$97k wages plus 401(k) distributions and partnership income) or ran an ABD/SSI-style resource test on the parents' ~$115k of bank and stock assets, which turned a zero-income adult into a household that looked over the limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
+us,scenario_064,dependent1_medicaid_eligible,31,prompt_ambiguity,household_unit_or_filing_status,False,,"The engine places dependent1 (age 27, no income of their own) in the MAGI adult category. It measures their Medicaid income at 0.00× FPL, so the head's wages, 401(k) distributions and partnership income are not counted against them. With zero countable MAGI, dependent1 is far below Wisconsin's adult income limit. The MAGI adult category has no asset test, and having employer coverage does not bar Medicaid, so the answer is eligible. Nearly every model assumed the parents' roughly $100k+ tax-unit income counted toward dependent1. Several also wrongly applied SSI-related asset limits or treated the ESI as disqualifying. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
us,scenario_064,dependent1_medicare_eligible,6,llm_error,age_disability,False,,"A person under 65 qualifies for Medicare only after 24 months of entitlement to Social Security Disability Insurance (SSDI), or through end-stage renal disease (ESRD) or ALS. Having a disability flag alone does not qualify anyone. Dependent 1 is 27. The household lists no SSDI or other Social Security benefits and no months of SSDI receipt, and all unlisted numeric inputs are 0, so Dependent 1 has no path to Medicare. The five models that answered treated the ""is disabled"" flag as Medicare eligibility and skipped the SSDI-entitlement step. The sixth model gave no usable answer."
us,scenario_064,dependent1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent2_chip_eligible,10,llm_error,health_coverage,False,,"Dependent 2 fails CHIP on two independent grounds. First, Dependent 2 is enrolled in employer-sponsored insurance, and CHIP excludes a child already covered by a group health plan. Second, household income is 321.5% of the federal poverty guideline, above Wisconsin's 306% CHIP limit. Every model that answered checked only the age-under-19 test and an income test it had underestimated (273-300% FPL), and none applied the rule that existing ESI coverage disqualifies the child. Dependent 2 is also not Medicaid-eligible, so no Medicaid pathway rescues the answer."
us,scenario_064,dependent2_medicaid_eligible,6,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent 2 is 18, so the only pathway open is the older-child MAGI category. It is income-tested against the five-person tax household's MAGI, which the engine puts at 3.22× FPL (322%). That is above Wisconsin's Medicaid limit for ages 6–18, above the roughly 306% BadgerCare/CHIP ceiling the models quoted, and above that ceiling plus the 5-point disregard (about 311%). No other pathway applies: there is no disability, pregnancy, caretaker status or SSI. Every model that answered Yes assumed or computed household MAGI below about 300–306% FPL and treated the CHIP ceiling as the Medicaid child limit. Its income measure fell short of the engine's 322%. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 0.00; the alternative is 1.00. The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household. Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
us,scenario_064,dependent2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent2_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"The traditional-IRA above-the-line deduction adds up traditional_ira_contributions for every tax-unit member. That puts Dependent 1's $18 contribution, made with $0 compensation, into the joint return's $126.22 IRA adjustment, which conflicts with IRC §219(a)/(c) (only the contributor, and a spouse on a joint return, can deduct) and §219(b)(1)(B) (the deduction is capped at includible compensation); dropping it raises the value by about $2.19 to about $4,441.48.","The answer depends on 2026 law after the One Big Beautiful Bill Act (OBBBA). AGI is $114,403.37: wages after the 401(k) deferral, plus $19,350 of partnership income (which already includes the $14,000 of self-employment partnership income), plus the $20,000 distribution, minus the $16,800 farm loss and a capital loss capped at $3,000. From that come the $32,200 MFJ standard deduction, the new qualified overtime deduction for the $13,899 FLSA premium, and a $510 QBI deduction (20% × $2,550), giving $67,794.08 taxable and $7,639.29 of tax. Then $2,200 CTC plus $500 ODC for each of the two older dependents is subtracted, leaving $4,439.29. There is no SE tax because the farm loss exceeds the SE partnership income, and the DPAD is repealed. Most misses came from leaving out the overtime deduction, counting the $14,000 of SE income twice, taking the repealed DPAD, or assuming TCJA sunset rules. The three $4,441.56 answers differ only because they left Dependent 1's $18 IRA contribution out of the tax unit's $126.22 IRA adjustment. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,439.29; the corrected value is 4,441.46. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)."
+us,scenario_064,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"The reference starts with AGI of $114,421.41. That is wages net of the $2,315 401(k) deferral, plus the $19,350 partnership/S-corp income (the $14,000 self-employment partnership amount is part of it, not added on top), plus the $20,000 distribution, minus the $16,800 farm loss, minus $3,000 of capital loss, minus the $108 IRA contribution. It then subtracts the 2026 OBBBA $32,200 standard deduction, the $13,899 FLSA overtime deduction and a $510 QBI deduction (20% of $19,350 − $16,800). That gives $67,812.12 taxable income and $7,641.46 of tax; the $2,200 CTC and two $500 other-dependent credits leave $4,441.46. Because the farm loss more than offsets the $14,000 SE partnership income, there is no SE tax or half-SE deduction, and the domestic production deduction was repealed after 2017. The wrong answers mostly went astray in a few ways: they left out the overtime deduction, counted the $14,000 twice, took the repealed DPAD or a half-SE-tax deduction, applied the pre-TCJA sunset rules, or added amounts that are not part of this measure."
us,scenario_064,federal_refundable_credits,6,llm_error,thresholds_rates,False,,"The refundable child credit is a residual, not an automatic payment: the $2,000-per-child CTC first offsets federal income tax, and only the portion left unused is refundable, subject to the $1,700-per-child ceiling and 15% of earned income above $2,500. This MFJ household's ~$98,810 of income (wages $97,295, $20,000 of 401(k) distributions, $19,350 partnership income, against a farm loss and a $3,000-capped capital loss) produces federal income tax before refundable credits far above $2,000, so the single under-17 child's entire CTC is absorbed nonrefundably, and the 18- and 27-year-old dependents generate only the $500 nonrefundable other-dependent credit. EITC is zero because income is multiples of the MFJ one-child phaseout ceiling, and no refundable AOTC (no students), recovery rebate, or refundable payroll tax credit applies, leaving $0. Every wrong model reported either the full CTC or the $1,700 ACTC ceiling as if refundability were automatic rather than conditional on unused credit."
us,scenario_064,free_school_meals_eligible,2,llm_error,taxable_income_or_deductions,False,,"The National School Lunch Program income test counts gross income by source — wages plus taxable retirement distributions — with no netting of farm losses, capital losses, or other tax-code offsets, and no deduction for premiums or medical expenses. PolicyEngine's countable income is $97,295 in wages plus $20,000 in taxable 401(k) distributions = $117,295 against an SPM-unit FPG of $38,680, a ratio of 3.03, far above the 130% free line ($50,284) and the 185% reduced-price line ($71,558). Wisconsin has no universal free meals and no SNAP/TANF categorical eligibility exists here, so the household lands in the PAID tier. The trap is importing tax-return netting (the -$16,800 farm loss and -$18,235 long-term capital loss) into a gross-income benefit test."
us,scenario_064,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -529,7 +527,7 @@ us,scenario_064,head_medicaid_eligible,1,parse_contract_failure,missing_output,F
us,scenario_064,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee-side FICA applies to the head's full $97,295 of gross wages: 6.2% Social Security ($6,032.29, wages far under the cap) plus 1.45% Medicare ($1,410.78) = $7,443.07, with no Additional Medicare Tax (wages under $250,000 MFJ) and no Wisconsin employee payroll tax, since Wisconsin levies no mandatory employee contribution. The trap is the FICA wage base: elective 401(k) deferrals are wages for FICA under IRC §3121(v)(1)(A), the $19,648 employer-sponsored insurance figure and the $1,560 non-Part-B premium do not reduce the modeled Social Security/Medicare base, the $13,899 FLSA overtime premium is already inside the $97,295 gross-wage total, and the partnership, self-employment, farm, capital-gain, and 401(k)-distribution amounts carry no employee FICA (self-employment tax is a separate excluded output). Every wrong dollar answer either shrank that base with a pre-tax deduction, inflated it with non-wage or already-included income, or bolted on a tax the output definition excludes."
+us,scenario_064,payroll_tax,18,llm_error,payroll_tax_base,False,,"The reference applies the employee FICA rates to the head's full listed gross wages: 6.2% × $97,295 = $6,032.29 and 1.45% × $97,295 = $1,410.78, for a total of $7,443.07. Wisconsin has no employee state payroll tax, wages are far below the Additional Medicare Tax threshold, and nobody else in the household has wages. Most models went wrong in one of two ways. Some made up reductions to the FICA base, either by treating the listed ESI or health premiums as Section 125 salary reductions or by subtracting traditional 401(k) deferrals, which are FICA wages under IRC 3121(v)(1)(A). Others reached $7,443.07 in their reasoning but then submitted a different number."
us,scenario_064,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,self_employment_tax,11,llm_error,payroll_tax_base,False,,"The trap is the aggregation rule for net earnings from self-employment: section 1402(a) combines all of a person's trades and businesses, so the head's -$16,800 farm operations loss is netted against the $14,000 of partnership self-employment earnings, leaving non-positive net earnings and therefore no self-employment tax. A second trap is that the $19,350 partnership/S-corp line is not itself SE earnings — the $14,000 is the SE-earnings portion carved out of that distributive share, and S-corp and limited-partner shares are excluded under section 1402(a)(13) — so adding the two double-counts the same K-1 income. Every wrong model applied 92.35% x 15.3% to a positive base, either by dropping the farm loss entirely or by stacking the two partnership lines so the loss could not zero it out. Three models returned no value at all for this key."
us,scenario_064,snap,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -538,34 +536,35 @@ us,scenario_064,spouse_medicaid_eligible,1,parse_contract_failure,missing_output
us,scenario_064,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,ssi,19,prompt_ambiguity,age_disability,False,,"Every substantive wrong answer assumes that Dependent 1's generic ""is disabled"" flag satisfies SSI's aged/blind/disabled requirement. In the engine, is_ssi_aged_blind_disabled is False for all five members: Head is 48, Spouse 47, Dependent 1 27, Dependent 2 18 and Child 1 12. Because no member passes that test, SSI is $0 before any income, resource, deeming or living-arrangement test applies. The models' later steps (the full federal benefit rate, the one-third or PMV reduction, the Wisconsin supplement, the parental-deeming analysis) all compute a benefit for a person the engine does not count as SSI-disabled. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support."
-us,scenario_064,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The engine starts from Wisconsin AGI of $114,403 and subtracts the MFJ sliding-scale standard deduction, which is phased down to $8,759 at this income, plus 5 personal exemptions of $700 each ($3,500). That leaves $102,144 of taxable income, taxed on the 2026 MFJ schedule. In that schedule the 4.4% bracket was widened by the 2025 budget and runs to about $50,500, so 5.3% applies only above that. No nonrefundable credit applies: the spouse has no earnings, so there is no married-couple credit, and itemized deductions fall below the standard deduction, so there is no itemized-deduction credit. Models went wrong in five ways: they overstated or understated WI AGI (double-counting partnership income, taking off the overtime premium, or adding back capital losses), mis-sized the phased standard deduction, dropped exemptions or treated them as credits, used the old narrower 4.4% bracket or stale rates, or invented nonrefundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,606.00; the corrected value is 4,599.62. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)."
+us,scenario_064,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Wisconsin AGI equals the federal AGI of $114,421.41. That figure is wages minus the $2,315 traditional 401(k) deferral, plus the $20,000 distribution and $19,350 partnership income, minus the $16,800 farm loss, minus $3,000 of capital loss (Wisconsin also allows $3,000 on a joint return), minus the $108 IRA deduction. There is no SE-tax deduction because the farm loss more than offsets the $14,000 SE partnership income. Subtracting the $8,953.27 sliding-scale standard deduction and five $700 exemptions leaves $101,968.14. That amount is taxed at 3.5%/4.4%/5.3% under the 2026 schedule, where the 4.4% bracket runs to roughly $69,000 for joint filers, and no nonrefundable credits apply. The models went wrong in five main ways: an outdated $500 capital-loss cap, the pre-2025 brackets where 5.3% starts near $38,000, a mis-sized sliding standard deduction, missing or misapplied exemptions, and made-up nonrefundable credits."
us,scenario_064,state_refundable_credits,4,llm_error,state_local_rule,False,,"Wisconsin's refundable individual income tax credits are the Homestead Credit, the Wisconsin EITC, the Farmland Preservation Credit, and the Veterans and Surviving Spouses Property Tax Credit; this household qualifies for none, so wi_refundable_credits and therefore state_refundable_credits are $0. The WI EITC is a flat percentage of the federal EITC (4%/11%/34% by number of qualifying children), and the federal EITC is $0 at $97,295 of wages plus partnership and 401(k) distribution income, so the WI EITC is $0. The Homestead Credit is barred both by its household-income ceiling near $24,680 and by the absence of any rent or property tax paid (only a mortgage balance is listed), and the Farmland Preservation Credit requires a farmland preservation agreement or certified zoning, which is unlisted and therefore false. The trap is arriving at an exact $0 rather than assuming that a household with a farm loss and Wisconsin residency picks up some small residual refundable credit."
us,scenario_064,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_066,federal_refundable_credits,9,llm_error,credit_phaseout,False,,"The childless EITC phases in from the first dollar of earned income at 7.65% with no minimum-earnings floor and no earned-income ""kick-in"" threshold, so $520 of wages for a 40-year-old single filer (inside the 25–64 age window, no disqualifying investment income) yields exactly 0.0765 × $520 = $39.78 — the reference value. Every wrong answer turns on inventing a floor that does not exist in IRC §32 or denying the childless-worker category outright; the phase-in is linear and unrounded in PolicyEngine, so the correct answer is a one-line multiplication. The $14,000 bank balance is irrelevant because no interest is listed and the §32(i) investment-income limit ($12,200-scale in 2026) applies to income, not assets. The only non-zero refundable component is EITC: no children rules out refundable CTC, and no education expenses rule out the refundable AOTC."
-us,scenario_066,head_medicaid_eligible,9,llm_error,health_coverage,False,,"Virginia's ACA adult expansion covers this 40-year-old nondependent because annual MAGI is 0.03 times FPL, below the MAGI limit. Other means-tested health coverage does not disqualify the person, MAGI-based expansion eligibility has no asset test, and the stated $520 annual wages control instead of annualizing the separate hourly-rate and weekly-hours facts."
-us,scenario_066,snap,31,llm_error,thresholds_rates,False,,"Two steps separate the reference from the wrong answers. First, Virginia's broad-based categorical eligibility (TANF non-cash) makes the household categorically eligible despite its $14,000 in bank assets, so the standard $2,750–$3,000 SNAP resource limit does not apply. Second, $43.33 of monthly earnings is wiped out by the 20% earned income deduction and the standard deduction, so net income is $0 and the benefit is the full FY2026 one-person contiguous-US maximum of $298 a month, or $3,576 a year. Seven models denied eligibility by applying the plain asset limit. Most of the rest got eligibility and zero net income right but used a stale or guessed maximum allotment ($275–$294, or $300) instead of $298, and a few subtracted a spurious 30% income contribution or applied a nonexistent time limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14))."
-us,scenario_066,state_refundable_credits,34,llm_error,state_local_rule,False,,"The $7.96 is Virginia's refundable EITC at 20% of the federal EITC. The federal childless EITC phases in at 7.65% from the first dollar, so $520 × 7.65% = $39.78, and $39.78 × 20% = $7.96. The credit is paid whether or not there is any state tax liability. Models fell into two traps. Most answered $0 because they denied that a Virginia refundable EITC exists, called it nonrefundable, said it had expired, or tied it to having tax liability. The models that did find the credit used the older 15% rate, which gives $5.97 or $6."
+us,scenario_066,federal_refundable_credits,10,llm_error,credit_phaseout,False,,"The childless EITC has no minimum earned-income floor. It phases in at 7.65% from the first dollar of earnings, and a 40-year-old single filer falls inside the 25–64 age window. With $520 of wages and no investment income, the credit is 0.0765 × $520 = $39.78, and the other refundable credits are all zero. Nine models wrongly assumed the EITC needs a minimum or ""substantial"" earned income, or that it depends on tax liability, so they answered $0. One model got $39.78 and then replaced it with a made-up $234 from a table lookup."
+us,scenario_066,head_medicaid_eligible,10,llm_error,categorical_eligibility,False,,"Virginia expanded Medicaid, so a 40-year-old non-disabled citizen qualifies in the ACA adult expansion category if MAGI is at or below 138% FPL. Here MAGI is the actual $520 of wages, about 0.03x FPL. Eligibility does not depend on current enrollment: the 'other means-tested health coverage at interview' flag does not disqualify the head. The MAGI adult category has no asset test, so the $14,000 bank balance is irrelevant. The hourly rate and weekly hours do not replace the stated annual wages. Most models said No because they treated current coverage as a bar, applied an asset limit, or annualized $32/hour × 40 hours into income the head never earned."
+us,scenario_066,snap,32,llm_error,thresholds_rates,False,,"This one-person Virginia household is categorically eligible for SNAP through Virginia's broad-based categorical eligibility, which uses the TANF non-cash tests. That test does not disqualify $14,000 in bank assets, so the federal $3,000 resource limit never applies. Earned income is $43.33 a month, so after the standard and shelter deductions net income is $0 and the household receives the full one-person maximum allotment of $298 a month (the FY2026 contiguous-US figure), or $3,576 a year. Models failed in two main ways. Some applied the federal asset limit and gave $0. Others used a stale or guessed maximum allotment, such as $292 (the FY2025 figure), $291, $294, $297 or $300, instead of $298. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14))."
+us,scenario_066,state_refundable_credits,35,llm_error,categorical_eligibility,False,,"With $520 of wages and no children, the head gets a federal EITC in the phase-in range: 7.65% × $520 = $39.78. Virginia's refundable EITC pays a share of the federal EITC whether or not the filer owes any Virginia tax, and since tax year 2024 that share is 20%, not the older 15%. That gives 20% × $39.78 = $7.96. Most models wrongly said Virginia has no refundable EITC, that it had expired, or that the federal EITC was zero. The models that did find the credit used the outdated 15% rate and got $5.97–$6."
us,scenario_067,dependent1_medicaid_eligible,20,prompt_ambiguity,household_unit_or_filing_status,False,,"PolicyEngine puts dependent1, age 23, in Indiana's ACA adult expansion group with a MAGI income level of 0.00×FPL. The parents' $82,360 in wages and $4,942 in pension never enter dependent1's Medicaid income measure, and dependent1's own $10,800 in disability benefits are not MAGI income. So dependent1 is far below the 138% FPL limit and eligible. All 20 models fell into the same trap in some form. Most added the parents' roughly $87K–$98K to a 3-person ""household MAGI"" of about 317% FPL. A few applied SSI-style or parental income and asset tests that the ACA adult category does not use, or counted the disability benefits as income that fails the limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
-us,scenario_067,dependent1_medicare_eligible,9,llm_error,age_disability,False,,"Dependent 1 is 23, so Medicare eligibility needs the disability pathway. Under 42 U.S.C. 426(b), that pathway requires 24 months of entitlement to Social Security disability benefits (or end-stage renal disease). The household lists only a generic 'disability benefits' amount and an 'is disabled' flag. It records no Social Security Disability Insurance entitlement, no months of SSDI receipt and no ESRD, and the prompt says every unlisted status is false. Every model treated 'disabled plus disability benefits' as enough for Medicare, so all nine answered Yes where PolicyEngine returns No."
-us,scenario_067,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The reference comes from a simple chain under 2026 law, where OBBBA made the TCJA structure permanent. AGI is $87,302 (wages plus pension). The dependent's $10,800 disability benefits stay off the joint return. The $32,200 MFJ standard deduction applies, with no auto-loan interest deduction because the facts establish no qualified vehicle loan. That leaves taxable income of $55,102, taxed at 10% up to $24,800 and 12% above for $6,116.24, less the $500 nonrefundable credit for the 23-year-old disabled dependent. The models split into groups: some applied a TCJA sunset (personal exemptions, 15% bracket, lower standard deduction); others used stale 2024/2025 standard deductions and bracket edges, took the auto-loan deduction, added the dependent's benefits to AGI, or mishandled the $500 dependent credit."
+us,scenario_067,dependent1_medicare_eligible,10,llm_error,categorical_eligibility,False,,"Before age 65, Medicare's disability pathway requires 24 months of entitlement to Social Security Disability Insurance, or end-stage renal disease or ALS. Being disabled is not enough on its own. In this household, Dependent 1 has generic ""disability benefits,"" which is a separate income input from Social Security disability. The months of SSDI receipt are unlisted, so they count as 0, and the prompt says not to infer unlisted benefit receipt. Dependent 1 is 23 and has no ESRD, so the person is not Medicare-eligible. Every model treated the disabled flag plus the generic disability income as SSDI entitlement with the waiting period already met."
+us,scenario_067,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference uses OBBBA's permanent TCJA structure for 2026: a $32,200 MFJ standard deduction, a 10% bracket up to $24,800, then 12%, and no personal exemptions. AGI is $87,302 because the dependent's $10,800 disability benefits are excluded and no auto-loan-interest deduction applies, so taxable income is $55,102 and tax is $6,116.24. The $500 nonrefundable credit for other dependents applies to the permanently disabled 23-year-old, who counts as a qualifying child because disability waives the age test, and that brings the result to $5,616.24. The wrong answers mostly came from one of these: applying the TCJA-sunset regime that OBBBA eliminated, using stale 2025 standard-deduction and bracket figures, deducting the listed auto loan interest, putting the dependent's disability benefits into AGI, imputing mortgage interest, or getting the $500 dependent credit wrong (leaving it out or swapping in a $2,000+ CTC)."
us,scenario_067,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The 23-year-old disabled household member does not generate a refundable Child Tax Credit because the CTC requires a qualifying child under age 17. With no CTC-qualifying child and no eligibility for the other refundable credit components, federal refundable credits equal $0."
us,scenario_067,head_medicare_eligible,1,llm_error,age_disability,False,,"Ordinary age-based Medicare eligibility begins at age 65, and the 60-year-old head is below that threshold. No qualifying disability-based Medicare pathway or other exception is supplied, so the head is not Medicare eligible."
-us,scenario_067,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee-side payroll tax here is a single, closed computation: 6.2% Social Security plus 1.45% Medicare on the head's $82,360 of wages — $5,106.32 + $1,194.22 = $6,300.54 — with no Additional Medicare Tax (wages far below $200,000/$250,000) and no Indiana mandatory employee payroll tax. The trap has two halves: the wage base is exactly the one listed wage figure (the spouse's 50 weekly hours carry no listed wage and are 0 under the ""treat any unlisted numeric input as 0"" instruction; the head's $67 hourly rate imputes nothing extra), and non-wage receipts — the head's $4,942 taxable pension and the dependent's $10,800 disability benefits — are not FICA wages under IRC §3121(a). The second half is precision: the answer is a cents-exact product, so truncating components or ""rounding"" the total is itself the error. Every wrong model either inflated the base with unlisted or non-wage income, or named the correct rule and then submitted a number its own arithmetic does not produce."
+us,scenario_067,local_income_tax,47,prompt_ambiguity,state_local_rule,False,,"The prompt defined local_income_tax by listing only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. PolicyEngine's local_income_tax also includes Indiana county income tax. Because no county was given, the engine used its default, Adams County, at 1.6% of Indiana AGI: $87,302 federal AGI minus $3,000 of exemptions for three people is $84,302, and $84,302 × 1.6% = $1,348.83. The prompt supports two readings: the list is exhaustive, so an Indiana household owes $0, or it gives examples of every local income tax the engine models, so Indiana county tax counts. Almost every model took the exhaustive reading and answered $0. Three models wrongly said Indiana has no local income tax at all, and one model recognized the county tax but used the wrong amount. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
+us,scenario_067,payroll_tax,6,llm_error,payroll_tax_base,False,,"The head is the only person with wages. Their $82,360 is taxed at 6.2% for Social Security ($5,106.32) and 1.45% for Medicare ($1,194.22), for $6,300.54 total. Those wages are below the 2026 Social Security wage base and the $250,000 Additional Medicare Tax threshold, and Indiana has no employee state payroll tax. The spouse has no listed wages, so the spouse's wages are $0, and neither the pension nor the disability benefits are FICA wages. The models went wrong by adding wages for the spouse, putting non-wage income into the tax base, or making rounding and arithmetic errors in the simple 7.65% calculation."
us,scenario_067,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_067,ssi,17,prompt_ambiguity,age_disability,False,"The question lists Dependent 1 (age 23) as ""is disabled,"" and that person has no earnings, so there is no substantial gainful activity to disqualify them; under 42 U.S.C. §1382c(a)(1)(A) and (a)(3) and 20 CFR 416.202(a), that person is categorically eligible for SSI, yet the engine computes is_ssi_aged_blind_disabled=False and the derivation says no member is flagged disabled in the engine's inputs, so the displayed disability flag is not the input that feeds is_ssi_disabled (an input-mapping mismatch).","The deciding step is SSI's aged/blind/disabled gate. The engine evaluates is_ssi_aged_blind_disabled=False for all three members: the head (60), the spouse (58), and Dependent 1 (23), who has no SSI disability status in the engine's inputs. That makes SSI $0 before any income, resource or deeming test runs. All 17 models treated Dependent 1's listed disability as meeting the SSI disability criterion and subtracted the $900/month disability benefit, less the $20 general exclusion, from the 2026 federal benefit rate (FBR), which lands near $1,368. The outliers come from separate mistakes: an old or invented FBR, or applying the $20 exclusion once a year instead of every month. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_data_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
-us,scenario_067,state_income_tax_before_refundable_credits,38,llm_error,state_local_rule,False,,"Indiana starts from federal AGI of $87,302 (wages plus taxable pension; the dependent's disability benefits are not in the filers' AGI) and has no standard deduction. It subtracts $1,000 personal exemptions for head, spouse, and dependent, for $3,000 total, and taxes the resulting $84,302 at the 2026 flat rate of 2.95%, giving $2,486.91. The $1,500 additional dependent-child exemption applies only to a child under 19 or a full-time student under 24, so it does not cover a 23-year-old disabled non-student. The wrong answers split into two groups. Some models used a stale or guessed rate (3.0%, 3.05%, 2.9%, 3.12%). Others added a $1,500 disabled-dependent exemption or a $1,500 dependent exemption, which takes exemptions to $4,500 or $5,000. A smaller group invented a standard deduction, pension deduction, or credits, or included the dependent's disability benefits in AGI."
+us,scenario_067,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Indiana starts from federal AGI of $87,302 (wages plus taxable pension; the dependent's disability benefits are not in the filers' AGI). It allows only three $1,000 personal exemptions, for the head, spouse and the 23-year-old dependent, totaling $3,000. Indiana has no standard deduction and no extra exemption for a disabled dependent. The $1,500 additional dependent-child exemption applies only to a child under 19 or a full-time student under 24, so this dependent does not get it. That leaves $84,302, taxed at the 2026 statutory flat rate of 2.95%, for $2,486.91. Most models used an outdated or future rate (3.05%, 3.0%, 2.9% or 2.75%), invented a $1,500 disabled-dependent or qualifying-child exemption, or added deductions Indiana does not allow."
us,scenario_067,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_068,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"PolicyEngine taxes only the $34,083.53 of listed wages. The separately listed $73,000 of educational assistance is not income, and the ESI premiums are not subtracted from the stated gross wages. Tax year 2026 runs under the permanent post-OBBBA TCJA structure: a $16,100 single standard deduction, 10% up to $12,400 and 12% up to $50,400. That gives $1,240 + 12% x $5,583.53 = $1,910.02, with no nonrefundable credits. The wrong answers fall into five groups: turning the educational assistance into $67,750-$73,000 of extra wages, assuming a TCJA sunset (personal exemption and 15%/25% brackets), subtracting the $8,389 ESI premium from wages, using stale 2024/2025 deductions and bracket edges, or wiping out the tax with an EITC or CTC this childless filer does not get."
+us,scenario_068,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"AGI is just the $34,083.53 in gross wages. The $73,000 of educational assistance is excluded from income, and the employer-sponsored insurance premiums are not subtracted from the gross wages. Under 2026 law, which keeps the TCJA structure permanent through OBBBA, the single standard deduction is $16,100. That leaves taxable income of $17,983.53, taxed at 10% up to $12,400 and 12% above it: $1,240 + $670.02 = $1,910.02. No nonrefundable credits apply. The wrong answers come from four main mistakes: adding most or all of the educational assistance to income, taking the ESI premiums out of wages, using pre-TCJA sunset parameters or stale 2024/2025 parameters, and subtracting the refundable EITC from a figure that comes before refundable credits."
us,scenario_068,federal_refundable_credits,4,llm_error,credit_phaseout,False,,"The decisive rule is the 2026 EITC income limit for a single filer with no qualifying children. At $34,084 of wages, the head is above the childless EITC phaseout endpoint, and the household has no qualifying child or other facts generating any refundable federal credit, so every component is zero."
us,scenario_068,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Maryland Medicaid eligibility for this 31-year-old requires qualification through a specific MAGI or non-MAGI category. The head's MAGI income is 2.14 times FPL, above the applicable adult MAGI limit, and the head qualifies through no non-MAGI pathway, so the eligibility output is 0."
us,scenario_068,payroll_tax,14,llm_error,payroll_tax_base,False,,"PolicyEngine's payroll_tax here is simply the employee FICA pair on the head's $34,084 of employment income: 6.2% Social Security ($2,113.18) plus 1.45% Medicare ($494.21), with the small cent-level offset from the engine's float32 wage representation. The trap has three parts: the FICA wage base is gross wages with no reduction for the $8,389 employer-sponsored insurance premium (PolicyEngine applies payroll tax to employment_income, not to a section 125 net-of-premium wage), the $73,000 educational assistance is student financial assistance rather than employment compensation and never enters the FICA base, and Maryland imposes no mandatory employee payroll contribution in 2026 — the FAMLI employee contribution is not in effect and is not part of this output. Every wrong answer comes from adding one of those three phantom adjustments, or from arithmetic that contradicts the model's own stated components."
-us,scenario_068,state_income_tax_before_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"Maryland's 2025 Budget Reconciliation and Financing Act replaced the 15%-of-income standard deduction and its roughly $2,350–$2,900 single cap with a flat $3,350 single standard deduction. Most models that used wages only kept the repealed capped formula, which overstated taxable income by about $450–$1,000. A second group counted the $73,000 of educational assistance as income (all of it, or the part above $5,250), or subtracted the $8,389 employer insurance premium from the listed gross wages. Maryland AGI here is only the $34,083.53 of wages. The correct computation is $34,083.53 − $3,350 − $3,200 = $27,533.53 of taxable income, and $90 + 4.75% × $24,533.53 = $1,255.34."
+us,scenario_068,state_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"Maryland AGI is just the $34,083.53 of wages. The $73,000 of educational assistance stays out of income, and the employer-sponsored insurance premiums do not reduce wages. Starting in 2025, Maryland replaced the old standard deduction (15% of income with a minimum and maximum) with a flat $3,350 for single filers, and the full $3,200 personal exemption applies. That gives $27,533.53 of taxable income: $90 on the first $3,000 plus 4.75% of the rest, or $1,255.34. Most models used the old capped deduction of $2,350 to $2,900, added $67,750 to $73,000 of educational assistance to income, subtracted the insurance premiums, or claimed deductions wiped out all the tax."
us,scenario_068,state_refundable_credits,2,llm_error,credit_phaseout,False,,"Maryland’s refundable EITC is tied to federal EITC eligibility, and a single 31-year-old filer with no qualifying children and $34,084 of wages is above the childless EITC phaseout ceiling. The federal EITC is therefore zero, producing no Maryland refundable EITC or other applicable refundable Maryland credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"Getting the reference takes two steps. First, apply the 2026 current-law parameters: TCJA rates were made permanent, the single standard deduction is $16,100 and the 10% bracket ends at $12,400. That gives $2,172.36 of tax on $20,169.67 of taxable income. Second, subtract the nonrefundable Saver's Credit at the 10% AGI tier on all $1,924 of retirement contributions, counting Roth 401(k) and Roth IRA with the traditional ones, which gives $1,979.96. The wrong answers fall into four groups: models that used a TCJA-sunset personal exemption and 15% bracket, models that used stale 2025-style standard deduction or bracket figures, models that left out the Saver's Credit entirely, and models that applied the wrong credit rate or dropped the Roth contributions from the credit base."
+us,scenario_070,federal_income_tax_before_refundable_credits,38,llm_error,credit_phaseout,False,,"The correct derivation has two steps. First, under the OBBBA-permanent TCJA structure for 2026, AGI of $36,269.67 minus the $16,100 single standard deduction gives taxable income of $20,169.67. Tax is 10% up to $12,400 and 12% above that, which comes to $2,172.36. Second, the nonrefundable Saver's Credit applies at the 10% tier for single filers at this AGI. It covers all $1,924 of elective deferrals and IRA contributions, both traditional and Roth, so it equals $192.40 and leaves $1,979.96. Models went wrong in three main ways: they left out the Saver's Credit, they assumed the TCJA expired (bringing back the personal exemption and the 15% bracket) or used stale 2024/2025 parameters, or they misapplied the credit's rate tier or contribution base."
us,scenario_070,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"The refundable Child Tax Credit requires at least one qualifying child; this household contains only a single 57-year-old adult. With no qualifying child and income above the childless EITC phaseout range, every refundable federal credit component is zero."
us,scenario_070,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Illinois’s ACA adult Medicaid pathway requires income no greater than 138% FPL, while the head’s MAGI income is 2.27 times FPL. The head also qualifies through no non-MAGI pathway, so the Medicaid category is NONE and eligibility is No."
us,scenario_070,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $38,000 of wages because the prompt lists $300 of health insurance premiums but never identifies them as a pre-tax cafeteria-plan deduction from FICA wages. The correct computation is $38,000 × 6.2% = $2,356 plus $38,000 × 1.45% = $551, totaling $2,907; the Additional Medicare Tax and Illinois employee payroll taxes are zero."
-us,scenario_070,state_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"Illinois tax here is 4.95% of federal AGI ($36,269.67, which already excludes the traditional 401(k) and deducts the IRA and student loan interest) less one personal exemption. For 2026 that exemption is inflation-indexed to $2,925, giving $33,344.67 of taxable income and $1,650.56 of tax. Most models got the AGI right but used a stale exemption: $2,850 (2025), $2,775 (2024), or $2,425 (pre-indexation). Each $75 of exemption shifts the tax by about $3.71. The rest broke the base itself: they left student loan interest out of AGI, subtracted health premiums, invented a standard deduction or renter's credit, applied the exemption twice, or dropped it entirely."
+us,scenario_070,state_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"Illinois tax here is 4.95% × (federal AGI − one personal exemption). Federal AGI is about $36,270: $38,000 wages less the $1,485 traditional 401(k), plus $100 interest, minus the $69 IRA deduction and $276 student loan interest. The 2026 exemption is $2,925, which gives $33,344.67 × 4.95% = $1,650.56. Most models got the base right but used an exemption from an earlier year ($2,425, $2,775 or $2,850), which pushes the tax a few dollars high. Others invented a standard deduction or credits, dropped the student loan interest adjustment, treated health premiums as pre-tax wage reductions, or zeroed out the liability."
us,scenario_070,state_refundable_credits,1,llm_error,credit_phaseout,False,,"Illinois's Earned Income Credit is calculated as a percentage of the federal EITC, so expanding the eligible age range does not create a credit when the federal EITC amount is zero. At this childless filer's income, the federal EITC has fully phased out, making the Illinois refundable credit zero."
us,scenario_071,head_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Age 76 does not itself establish Medicaid eligibility, and this person qualifies through none of the Medicaid pathways represented by the engine. Their MAGI income is 2.28 times FPL, they receive no SSI, and the engine assigns medicaid_category = NONE; Medicare eligibility, housing assistance, and the mere availability of aged or spend-down programs do not create Medicaid eligibility."
us,scenario_071,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to listed wages and other covered earnings, not to work hours by themselves. The household has no listed wage or self-employment income, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll tax bases are all zero, yielding payroll_tax of $0."
@@ -578,94 +577,94 @@ us,scenario_072,state_income_tax_before_refundable_credits,17,llm_error,state_lo
us,scenario_073,head_medicaid_eligible,8,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds the untaxed portion of Social Security benefits back into income. That means the full $22,646 of SSDI counts, on top of the $3,108 pension. The head has no wages, so the traditional 401(k) and IRA contributions reduce nothing. MAGI is about $25,754, or 1.60 times FPL, which is above Michigan's 138% FPL limit for the adult expansion group. No other category applies: the head is 57, gets no SSI, is not pregnant, has no dependent children and has no disability flag, so the engine assigns medicaid_category NONE. Six models wrongly left untaxed SSDI out of MAGI or simply called the income 'low'. One subtracted 401(k) deferrals the head cannot make without wages. One treated SSDI receipt on its own as a disability route into Medicaid."
us,scenario_073,head_medicare_eligible,18,prompt_ambiguity,age_disability,False,,"The head is 57, so the age-65 route is out. The only other route is disability-based Medicare, which starts only after 24 months of Social Security disability benefits (42 U.S.C. 426(b)). The household lists this year's SSDI amount but not how long the head has received it. The prompt sets every unlisted number to 0 and every unlisted status to false, so the months of SSDI receipt count as 0 and the waiting period is not met. That leaves the head not Medicare eligible. All 18 models made the same mistake: they treated this year's SSDI as proof of disability-based Medicare, either by skipping the 24-month rule or by assuming it was already met. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_073,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earnings, not Social Security disability benefits or taxable private pension income. With no wage, salary, or self-employment income listed, the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components are all zero."
-us,scenario_073,snap,43,llm_error,categorical_eligibility,False,,"This household is categorically eligible for SNAP through Michigan's TANF non-cash (BBCE) rules: gross income is 165% of FPL, under the 200% limit, and categorical eligibility waives both the asset test and the 100% FPL net income test. Its net income of $1,755 gives a 30% contribution of $527, more than the $298 one-person maximum. Because the household is eligible, the one-to-two-person minimum allotment of $24 a month still applies, for $288 a year. Most models either applied the 100% FPL net income test, or a 130% gross test, and declared the household ineligible, or they stopped at a computed benefit of zero without applying the minimum. A few applied the minimum but used the FY2025 $23 or a rounded $25, and two built large medical deductions or a wrong maximum allotment that produced a sizeable benefit."
+us,scenario_073,snap,44,llm_error,categorical_eligibility,False,,"The household has one disabled member (SSDI). Its monthly gross income is $2,146, about 165% of FPL, which is under Michigan's 200% FPL limit for broad-based categorical eligibility, so the household is categorically eligible through TANF non-cash benefits. That means the 100% FPL net income test does not apply. Net income is $1,755, so the formula benefit is $298 − 0.3 × $1,755, which is below zero. Categorically eligible one- and two-person households still get the minimum allotment, which is $24 a month (about 8% of the $298 one-person maximum), or $288 a year. Most models applied a net income test, a 130% gross test, or an asset limit and stopped at $0, or they never applied the minimum allotment. A few applied the minimum but used the outdated $23 or a wrong $25 amount."
us,scenario_074,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_074,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_074,head_medicaid_eligible,6,llm_error,health_coverage,False,,"Medicaid MAGI is not taxable income: 26 U.S.C. §36B(d)(2)(B) adds back to AGI all Social Security benefits excluded from gross income, so the head's $33,640 SSDI counts in full alongside the $384 taxable IRA distribution, producing 2.13 x FPL — roughly 213% of poverty against Louisiana's 138% FPL expansion-adult ceiling. Every wrong model shrank that base: four dropped SSDI entirely as ""non-taxable,"" one halved it using the §86 provisional-income 50% factor, and one asserted a disability pathway. The non-MAGI routes are also closed: the SSI-related aged/blind/disabled categories require SSI receipt (SSI = $0) or age 65+ (the head is 54), and the head is not a tax-unit dependent, so medicaid_category resolves to NONE."
-us,scenario_074,head_medicare_eligible,27,prompt_ambiguity,age_disability,False,,"Under-65 Medicare eligibility through disability requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)). PolicyEngine tracks the months of SSDI receipt as a separate input. The household facts list only a positive SSDI amount for the year, with no receipt duration, and the prompt says to treat unlisted numeric inputs as 0 and unlisted statuses as false. It also says facts are constant only across the one tax-benefit year, which covers at most 12 months of receipt. So the disability pathway fails, and eligibility falls back to the age-65 test, which the 54-year-old head does not meet. All 27 models treated current SSDI income as proof that the 24-month waiting period was already met and answered Yes. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
+us,scenario_074,head_medicare_eligible,28,prompt_ambiguity,age_disability,False,,"The head is 54, so the age-65 test fails. The only other route is the disability route, which requires 24 months of Social Security disability entitlement. PolicyEngine checks a count of months receiving SSDI. It does not check the SSDI dollar amount. The household gives no months of SSDI receipt and no disability status, and the prompt says unlisted numbers are 0 and unlisted statuses are false. So the 24-month requirement is not met and the head is not Medicare eligible. Every model treated a positive SSDI amount as proof that the waiting period was already done, which adds a fact the prompt never gave. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_074,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not to Social Security disability benefits, pension income, IRA distributions, or tax refunds. The household has no listed wages, and the prompt requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and the resulting payroll tax is $0."
-us,scenario_075,federal_income_tax_before_refundable_credits,27,llm_error,thresholds_rates,False,,"The correct 2026 computation uses the TCJA structure that OBBBA made permanent. That means a $16,100 single standard deduction, no personal exemption, and single brackets of 10% to $12,400, 12% to $50,400 and 22% to $105,700. This gives $13,843.28 of ordinary tax on $86,960, plus 15% on $6,226 of LTCG and qualified dividends ($933.86), for $14,777.15. No nonrefundable credit applies: at age 62 there is no senior deduction or age-65 add-on, and the Credit for the Elderly or Disabled phases out completely at $109,286 AGI. The wrong answers fall into four groups. Models that assumed the TCJA sunset (personal exemption and 10/15/25 brackets) landed near $17.5k-$18.9k. Models using the 2025 or guessed standard deductions and brackets landed near $14.9k-$15.2k. Two models computed $14,777 and then submitted a different number. The rest invented age or disability deductions or credits."
+us,scenario_075,federal_income_tax_before_refundable_credits,28,llm_error,thresholds_rates,False,,"The 2026 tax falls under the OBBBA, which made the TCJA structure permanent. That means a $16,100 single standard deduction, no personal exemption, and single brackets of 10% up to $12,400, 12% up to $50,400 and 22% up to $105,700. Ordinary taxable income of $86,960 is therefore taxed at $13,843, and the $6,226 of long-term gains and qualified dividends stacked on top is taxed at 15% ($934), for a total of $14,777. Nothing else applies: a 62-year-old gets no age-65 additional deduction or senior deduction, disability does not raise the standard deduction, and the Credit for the Elderly or Disabled is fully phased out at $109,286 AGI. The largest group of models assumed the TCJA sunset (personal exemption plus 10/15/25% brackets). Another group used 2025 or undersized 2026 standard deductions and bracket thresholds. A few computed $14,777 correctly but then submitted a different number."
us,scenario_075,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_075,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not establish Florida Medicaid eligibility. The head qualifies through none of Florida's Medicaid pathways, so the engine assigns medicaid_category NONE; the 6.85× FPL income level also rules out any applicable MAGI category."
us,scenario_075,head_medicare_eligible,15,llm_error,age_disability,False,,"Under age 65, Medicare disability eligibility depends on 24 months of Social Security Disability Insurance entitlement, or on end-stage renal disease. A general ""is disabled"" flag does not qualify anyone. This head is 62 and has no Social Security disability benefits: there are no months of SSDI receipt and no ESRD. Their income is wages, pensions, dividends and capital gains. So the age test (65+) fails and neither non-age pathway applies. Every model instead treated the disability flag, or age 62, as enough for Medicare."
-us,scenario_075,payroll_tax,4,llm_error,payroll_tax_base,False,,"Employee payroll tax applies only to the $12,656 of wages; pension, dividends, and capital gains are outside the employee FICA base. PolicyEngine separately computes $784.69 of employee Social Security tax and $183.52 of employee Medicare tax, totaling $968.21, with no Additional Medicare Tax or Florida employee payroll tax."
+us,scenario_075,payroll_tax,5,llm_error,other,False,,"Only the $12,656 in wages is subject to payroll tax. Pension income, dividends and capital gains are not FICA wages. Employee Social Security at 6.2% plus employee Medicare at 1.45% comes to about $968. Wages are far below the Additional Medicare Tax threshold, and Florida has no employee state payroll tax. No model was misled by the tax base. The errors are arithmetic slips, a final value that contradicts the model's own correct working, and missing or unparsed answers."
us,scenario_076,child2_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC serves children only through their fifth birthday and generally requires household income at or below 185% of the federal poverty guideline. The model also inverted the income comparison: $173,820 is far above 185% of the poverty guideline for a three-person household, not 184% of the guideline."
-us,scenario_076,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"The correct 2026 chain under post-OBBBA law runs as follows. AGI is $164,220: full $160,000 wages plus $4,000 rent plus $220 interest, with child support excluded and no reduction for the ESI premium. Subtract the $24,150 HoH standard deduction and the $800 QBI deduction on rental income to get taxable income of $139,270. The 2026 HoH brackets (10% to $17,700, 12% to $67,450, 22% to $105,700, 24% above) give $24,211.80 of tax. Subtracting the full $4,400 CTC ($2,200 x 2) and the $200 CDCC leaves $19,611.80. Models went wrong in several ways. Many assumed TCJA had expired, so they applied personal exemptions, 10/15/25 brackets and a phased-out CTC. Others used 2025 bracket thresholds or $2,000-per-child CTC. Some cut wages by the ESI premium or omitted the rental QBI deduction. Some treated most of the CTC as refundable even though tax liability far exceeds the credit, and a few imputed mortgage interest from a loan balance."
+us,scenario_076,federal_income_tax_before_refundable_credits,41,llm_error,thresholds_rates,False,,"The correct path uses AGI of $164,220. Wages are not reduced by the ESI premium, and child support is excluded. Subtract the 2026 head-of-household standard deduction of $24,150 and an $800 QBI deduction (20% of the $4,000 rental income) to get taxable income of $139,270. The 2026 head-of-household brackets (10% to $17,700, 12% to $67,450, 22% to $105,700, then 24%) give $24,211.80 of tax. The full $4,400 CTC ($2,200 per child) and the $200 CDCC are nonrefundable against that liability, leaving $19,611.80. The wrong answers came from a few mistakes: assuming the TCJA expired in 2026 (personal exemptions, a $1,000 CTC that phases out); using 2025 brackets or a $2,000-per-child CTC; dropping the QBI deduction; subtracting the ESI premium from wages; estimating mortgage interest that the prompt never listed; treating the refundable ACTC cap as shrinking the nonrefundable CTC; and using the wrong filing status."
us,scenario_076,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is not an entitlement of $1,700 per child; it is only the portion of the $2,200-per-child CTC that survives after the credit offsets tax liability, and it is further capped at 15% of earned income over $2,500 and at $1,700 per child. This head-of-household filer has AGI of $164,220 (wages $160,000 + rents $4,000 + interest $220; child support is not taxable), below the $200,000 HoH phase-out threshold, so the full $4,400 CTC is available — but taxable income near $140,000 after the 2026 HoH standard deduction and the QBI deduction produces federal tax well above $25,000, so the entire CTC is absorbed nonrefundably and nothing spills into refundable_ctc. EITC is zero on this income, and there is no AOTC, recovery rebate, or refundable payroll tax credit here, so federal refundable credits are $0. Every wrong model computed the ACTC cap and then reported the cap itself as a payment, skipping the tax-liability offset that defines the refundable portion."
us,scenario_076,payroll_tax,12,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are calculated on the full $160,000 of wages in the computation trace; the listed $6,480 employer-sponsored insurance premium is not subtracted from the payroll-tax base. This yields $9,920 of Social Security tax plus $2,320 of Medicare tax, for $12,240 total, with no Additional Medicare Tax or mandatory Idaho employee payroll tax."
us,scenario_076,self_employment_tax,2,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to net earnings from a trade or business, and the prompt lists no self-employment income. The $4,000 of rental income remains rental income and does not create a self-employment tax base, so the computation yields $0."
-us,scenario_076,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Idaho's 2026 tax is a flat 5.3%, but it applies only to taxable income above a zero-rate amount. For a head-of-household filer that amount is $9,622, the joint-level figure. Idaho taxable income also comes out lower than most models assumed: federal AGI of $164,220, minus Idaho's $1,000 household and dependent care subtraction, minus the $24,150 standard deduction, minus the $800 QBI deduction on rental income, gives $138,270. The result is ($138,270 − $9,622) × 5.3% = $6,818.34, with no further nonrefundable credit subtracted. Most models made at least one of these mistakes: they taxed all taxable income without the zero-rate amount, used the single-level threshold (about $4,900), used the old 5.695% or 5.8% rate, missed the $1,000 or $800 deductions, or subtracted a $410 Idaho child tax credit that the reference computation does not include. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount. Reference affirmed (Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601)."
-us,scenario_076,state_refundable_credits,39,llm_error,thresholds_rates,False,,"The whole $465 is Idaho's refundable grocery credit at the 2026 amount of $155 per person, times 3 household members (head plus two children), with 12 qualifying months each. The credit has no income test, so the household's $160,000 in wages does not reduce it. The wrong answers split three ways. Most models applied a stale per-person amount: $120 (the 2023-2025 level, giving $360), $100 (the pre-2023 level, giving $300) or an invented $170. Others declared the grocery credit nonrefundable or income-limited and answered $0. Two models swapped in Idaho's nonrefundable $205 child tax credit."
-us,scenario_077,federal_income_tax_before_refundable_credits,29,llm_error,thresholds_rates,False,,"The correct path is AGI = $26,006 wages minus the $3,000 capital-loss limit = $23,006. Under current law after the OBBBA, the 2026 single standard deduction is $16,100 and there is no personal exemption, so taxable income is $6,906. All of it falls in the 10% bracket, which gives about $691, and no nonrefundable credits apply. The models went wrong in four ways. Most used an outdated standard deduction: $15,000, $15,350, $15,400, $15,450 or $15,750. Others applied the TCJA-sunset rules ($8,300 standard deduction plus a $5,300 exemption). Several also subtracted the $8,389 ESI premium from the stated wages, which the reference does not do. A group of models asserted zero tax, as if the standard deduction exceeded AGI or unnamed credits wiped out the liability."
+us,scenario_076,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Idaho taxable income is $164,220 AGI minus the $24,150 HoH standard deduction, the $1,000 Idaho child-care subtraction and the $800 QBI deduction, which leaves $138,270. The 2026 flat 5.3% rate applies only above the indexed head-of-household zero-rate band of about $9,622, which gives $6,818.34. No Idaho nonrefundable credit is applied, and the $10 permanent building fund tax is then added. Most models taxed the whole amount at 5.3%, or at the outdated 5.695%/5.8% rates. Many also dropped the child-care subtraction or the QBI deduction, subtracted a $410 Idaho child tax credit that is not part of the 2026 computation, and left out the $10 permanent building fund tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount. Reference affirmed (Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601)."
+us,scenario_076,state_refundable_credits,40,llm_error,thresholds_rates,False,,"Idaho's state refundable credit here is the grocery credit. It has no income phase-out, is refundable, and is allowed for each personal exemption: the head and both children. For 2025 and later it is $155 per person under 65, so this household gets 3 × $155 = $465. The wrong answers fall into two groups. Models answering $360 or $300 used the old $120 or even older $100 per-person amounts. Models answering $0 wrongly treated the grocery credit as nonrefundable or income-limited, or missed it entirely."
+us,scenario_077,federal_income_tax_before_refundable_credits,30,llm_error,thresholds_rates,False,,"The correct path is short. AGI is $26,006 in wages minus the $3,000 cap on net capital losses, which gives $23,006. Under OBBBA, the TCJA structure is permanent, so the 2026 single standard deduction is $16,100 with no personal exemption. That leaves $6,906 of taxable income, all taxed at 10%, for $690.62. No nonrefundable credits apply. Most models went wrong in one of three ways: they used a stale or guessed standard deduction ($15,000, $15,350, $15,750, and so on), they applied TCJA-sunset rules ($8,300 standard deduction plus a $5,300 exemption), or they subtracted the ESI premiums from wages that were already stated as gross. Several others computed about $691 and then submitted a different number or zeroed out the tax for no reason."
us,scenario_077,federal_refundable_credits,13,llm_error,credit_phaseout,False,,"The only potentially applicable refundable credit is the childless EITC, and its phaseout uses the greater of earned income or adjusted gross income. The $26,006 of wages is earned income; employer-sponsored insurance premiums do not reduce that amount, and at this earnings level the childless EITC is fully phased out. With no qualifying child or student and no applicable rebate or payroll credit, every refundable component is zero."
us,scenario_077,head_medicaid_eligible,10,llm_error,taxable_income_or_deductions,False,,"Louisiana's expansion pathway requires MAGI at or below the applicable adult limit, but the engine computes the head's MAGI income level as 1.44 times FPL, so that pathway fails. The listed employer-sponsored insurance premium is not established as a pre-tax payroll deduction and cannot simply be subtracted from wages; after the applicable MAGI computation, the head qualifies through no Medicaid category."
us,scenario_077,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_077,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are computed from the applicable payroll-tax wage base, and the reference calculation applies the 6.2% and 1.45% employee rates to the full $26,006 of wages. The employer-sponsored insurance premium input does not reduce that wage base in this calculation, producing $1,612.38 of Social Security tax plus $377.09 of Medicare tax, or $1,989.47."
us,scenario_077,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_077,snap,5,llm_error,categorical_eligibility,False,,"Louisiana’s broad-based categorical eligibility gross-income screen does not itself guarantee a SNAP allotment; the household must still produce an eligible benefit under the net-income and allotment calculations. SNAP countable income does not subtract capital losses, a mortgage balance is not a shelter expense, and a non-elderly, nondisabled household cannot deduct medical expenses. With no listed deductible shelter payment and countable wages of $26,006, this one-person household receives $0."
-us,scenario_077,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"Louisiana's 2025 reform (Act 11 of the 2024 Third Extraordinary Session) replaced the graduated brackets and the $4,500 combined personal exemption with a flat 3% rate and a $12,500 single standard deduction for 2025. R.S. 47:294(B) raises it by the CPI-U increase for calendar 2025. The December 2025 index, released January 13, 2026, puts that at 2.68% (the Legislative Fiscal Office's figure in the Louisiana Register of February 20, 2026), which gives $12,835. The reference taxes $23,006.18 AGI − $12,835 = $10,171.18 at 3%, which gives $305.14. Before the reference freeze the Department of Revenue applied 2.7% to the retirement income exemption, which R.S. 47:44.1 indexes in the same words (Louisiana Register, June 20, 2026); on September 28, 2026 it set the standard deduction at $12,838 the same way (RIB 26-019). That gives $305.05, within the $1 tolerance for an exact match. The $12,875 in the Department's 2026 withholding tables and estimated-tax worksheet was a provisional amount from CPI-U data available on December 1, 2025. GPT-6 Astra's $303.93 used that $12,875, which it called inflation-adjusted, on the $23,006 AGI that the prompt's $26,006 of wages and the $3,000 capital-loss limit give; with $12,875 the engine gives $303.94. Of the 46 models, 19 used 2025's $12,500 and answered $315.18 or $315; three more reached about $315 that way and then submitted $446 to $496, and two paired $12,500 with the repealed 1.85% rate or the $4,500 exemption. Ten more applied repealed rules (a personal exemption, the 1.85%/3.5% brackets or a deduction for federal income tax), four of them after taking the $8,389 ESI premiums out of AGI. Seven answered $0, saying Louisiana has no income tax, that PolicyEngine does not model it, or that deductions or low income leave nothing to tax. The rest gave a figure with no derivation, subtracted an unstated $15,400, or gave no answer."
+us,scenario_077,state_income_tax_before_refundable_credits,47,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform (Act 11 of the 2024 special session) set a flat 3% rate and a $12,500 single standard deduction, and indexed that deduction for inflation starting in tax year 2026, which makes it $12,835. Federal AGI is $23,006 because net capital losses are capped at $3,000. Subtracting $12,835 leaves $10,171.18 of taxable income, and 3% of that is $305.14. The most common wrong answer, $315.18, came from using the unindexed 2025 deduction of $12,500. Most other misses applied repealed pre-2025 Louisiana law (1.85%/3.5% brackets, a $4,500 personal exemption, the deduction for federal income tax paid), took ESI premiums out of gross wages a second time, or said Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
us,scenario_077,state_refundable_credits,4,llm_error,state_local_rule,False,,"The models applied Louisiana’s former refundable earned income tax credit, equal to 5% of the federal EITC, to tax year 2026. Louisiana no longer provides that refundable credit in 2026, so the Louisiana refundable-credits computation returns zero regardless of an estimated federal EITC amount."
us,scenario_077,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_078,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The key step is the SALT deduction. It combines Maryland state income tax, Maryland county (local) income tax and the $26,460 real estate tax, and together they exceed the 2026 OBBBA cap of $40,400, so SALT is exactly $40,400. Add $28,210 of mortgage interest for $68,610 in itemized deductions. AGI of $200,136.91 minus $68,610 leaves taxable income of $131,526.91, and the 2026 TCJA-permanent single brackets (10/12/22/24%) give $24,164.46. The wrong answers fall into four groups: (1) the expired $10,000 SALT cap (~$31.4–31.5k); (2) treating TCJA as sunset, which restores personal exemptions and the 15/25/28% brackets (~$27–28.5k); (3) counting only real estate tax in SALT ($27,510.08); and (4) adding state income tax but leaving out county income tax, which keeps SALT below the cap (~$25.85k). Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_078,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"Under OBBBA rules for 2026, TCJA brackets and the $16,100 standard deduction stay in place, and the SALT cap rises to $40,400. There is no phase-down here because MAGI of about $200,137 is far below $505,000. Real estate taxes of $26,460 plus the Maryland state and county income tax on about $200k of wages exceed $40,400, so SALT is capped at $40,400. With $28,210 of mortgage interest, itemized deductions total $68,610 and taxable income is $131,526.91, which gives $24,164.46 at the 10/12/22/24% brackets (24% bracket starting at $105,700). The models went wrong in one of four ways: applying the old $10,000 SALT cap, assuming the TCJA sunset (personal exemptions and the 15/25/28% brackets), leaving Maryland income tax out of SALT entirely, or counting too little Maryland state-plus-county income tax so SALT stayed under the cap. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_078,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_078,local_income_tax,1,llm_error,state_local_rule,False,,"The requested local_income_tax output is limited to the separately modeled local taxes listed in the prompt: NYC, Philadelphia, Kansas City, and St. Louis. Maryland county income tax is not included in this output, so a Maryland household outside those modeled jurisdictions has a value of zero."
-us,scenario_078,payroll_tax,26,llm_error,payroll_tax_base,False,,"The whole case reduces to two federal components: 6.2% on the 2026 Social Security taxable wage base of $184,500 ($11,439) plus uncapped 1.45% Medicare on the full $198,505 ($2,878.32), for $14,317.32. Wages fall $1,495 short of the $200,000 single-filer Additional Medicare Tax threshold, so that piece is zero, and Maryland imposes no mandatory employee-side payroll tax for 2026 — FAMLI employee contributions do not begin in that year. The wrong answers fall into three groups: stale or invented Social Security wage bases ($137,700, $168,600, $176,100, $177,600, ~$181,800), fabricated Maryland FAMLI employee contributions of roughly 0.45% of wages, and submissions whose numeric value contradicts the model's own correct arithmetic."
-us,scenario_078,state_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"Maryland AGI subtracts the $1,632 state tax refund, leaving $198,505. The filer itemizes $28,210 of mortgage interest plus the full $26,460 of property tax, which is under the 2026 $40,400 SALT cap. Maryland's phase-out cuts itemized deductions by 7.5% of federal AGI over $200,000, a $10.27 reduction that leaves $54,659.73. The personal exemption is $0 for single filers above $150,000, so taxable income is $143,845.27. The 2%/3%/4%/4.75%/5%/5.25% brackets give $6,936.88 with no county tax. The wrong answers split into five errors: taking the $2,400-$3,350 standard deduction instead of itemizing, capping property tax at $10,000, skipping the refund subtraction, keeping a partial exemption, or adding county piggyback tax."
+us,scenario_078,payroll_tax,27,llm_error,thresholds_rates,False,,"The correct payroll tax uses the 2026 Social Security wage base of $184,500: 6.2% × $184,500 = $11,439, plus 1.45% Medicare on all $198,505 of wages ($2,878.32), for $14,317.32. Wages are below the $200,000 Additional Medicare Tax threshold, and Maryland charges no mandatory employee payroll tax in 2026. Most wrong answers used an outdated or wrongly projected wage base. Others added a Maryland FAMLI contribution, which is not collected in 2026, or invented a Maryland payroll tax. Several models worked out exactly $14,317.32 and then submitted a different number."
+us,scenario_078,state_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"Maryland AGI is federal AGI of $200,136.91 minus the $1,631.91 state tax refund subtraction, which leaves $198,505. Maryland itemized deductions are federal itemized deductions minus state income taxes. Here that means mortgage interest of $28,210 plus the full $26,460 in real estate taxes, since that amount is under the 2026 federal SALT cap of $40,400. This $54,670 total is reduced by 7.5% of federal AGI above $200,000 ($10.27), giving $54,659.73. A single filer above $150,000 gets no personal exemption, so taxable income is $143,845.27. Maryland brackets (2/3/4/4.75/5/5.25%) then give $6,936.88. Most wrong models did one of four things: took the small Maryland standard deduction instead of itemizing, capped property taxes at $10,000, left the refund in Maryland AGI or claimed an exemption, or added county income tax."
us,scenario_078,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_079,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The operative rule is categorical, not income-based: the head's blindness, disability, and $7,560 of SSDI leave countable income below the SSI federal benefit rate, producing $5,286 of SSI, and Arizona is a 1634 state where SSI receipt automatically confers Medicaid with no MAGI test. Both models ran an income/coverage screen instead of the SSI-recipient category, and neither recognized that SSI here is a computed output of the listed facts rather than an unlisted input. The screen they ran also points to eligibility rather than away from it — 0.35 x FPL is far under Arizona's 138% FPL expansion threshold — so 'No' is unreachable by either the categorical or the MAGI route."
us,scenario_079,head_medicare_eligible,24,prompt_ambiguity,age_disability,False,,"Medicare eligibility here has two routes: reaching age 65, or completing 24 months of Social Security disability entitlement (42 U.S.C. 426(b)). PolicyEngine checks the second route against a months-of-SSDI-receipt input. The household lists SSDI income but no months of receipt, and the prompt says to treat unlisted numeric inputs as 0 and not to infer unlisted facts, so the waiting period is not met. Even if Head received SSDI for all of 2026, that adds up to only 12 months. With Head aged 58, neither route applies, but all 24 models treated SSDI receipt or the disability flag as enough for Medicare, and several simply assumed the 24-month wait had already passed. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_079,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_079,snap,46,llm_error,state_local_rule,False,,"The reference makes two moves. First, it counts the couple's $881/month of SSI (the couple benefit reduced by $610 of countable SSDI) together with the $630 of SSDI, for $1,511 of gross unearned income. Second, it applies Arizona's $145 standard medical deduction for elderly/disabled households whose medical costs exceed $35/month, rather than the $26 actual excess ($61 − $35). No model applied the $145 Arizona deduction. Models that counted SSI came out about $36/month too high ($1,932–$2,136 a year). Models that dropped SSI inflated the benefit to $3,400–$6,600, and the zero answers invented asset, shelter, or SSI cash-out barriers that don't apply in Arizona."
+us,scenario_079,snap,47,llm_error,state_local_rule,False,,"Each month the household has $1,511 of countable unearned income: $630 of SSDI plus $881 of SSI for the couple, which the models had to work out themselves. Arizona gives elderly/disabled households with more than $35 a month in medical costs a $145 standard medical deduction instead of their actual excess, which here is only $26. With the $209 standard deduction, net income is $1,157, so the household's expected contribution is $348 and the benefit is $546 − $348 = $198 a month ($2,376 a year). The models split into three groups: about 15 left out the SSI income and landed near $4,500–$5,200; about 15 included SSI but used the $26 actual medical excess and landed near $1,900–$2,100; and the rest made up asset, income-limit, shelter, or SSI-exclusion reasons to answer $0."
us,scenario_079,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The spouse is blind and disabled with zero countable income and $1,045 in resources, so she receives SSI ($5,286 in the household trace); Arizona is a 1634 state, where SSI receipt confers automatic Medicaid through the SSI-recipient category with no separate income test. Both models evaluated only an income/MAGI-style adult pathway and never checked the SSI-recipient categorical route that the engine actually used (medicaid_category = SSI_RECIPIENT). The error is compounded because even the MAGI route qualifies here: household MAGI is 0.35 × FPL, far under Arizona's 138% FPL expansion-adult ceiling, and Indian Health Service coverage is not minimum essential coverage that bars Medicaid. Zero-income, SSI-receiving disabled adults in an expansion state are Medicaid-eligible on two independent pathways, so ""No"" is unreachable from the stated facts."
us,scenario_079,spouse_medicare_eligible,4,llm_error,age_disability,False,,"The spouse is 55, so the only route to Medicare is the disability pathway. That route needs 24 months of Social Security disability (SSDI) entitlement, or ESRD/ALS. Being disabled or blind does not qualify anyone for Medicare by itself. The spouse has no Social Security disability income at all (the $7,560 belongs to the head), and no months of SSDI receipt or ESRD are listed, so all of these inputs default to zero or false. Every model treated ""is disabled"" as enough for Medicare and skipped the SSDI entitlement and 24-month waiting-period requirement."
-us,scenario_079,ssi,33,llm_error,thresholds_rates,False,,"Both spouses are blind and disabled, so they form an SSI eligible couple. Countable resources are the $1,045 bank account, because the one vehicle and the home are excluded. Countable income is the head's $630/mo SSDI less the $20 general exclusion, or $610/mo. The 2026 couple federal benefit rate is $1,491/mo ($745.50 per spouse), so SSI is ($1,491 - $610) x 12 = $10,572. Most models got the $610 of countable income right but used the wrong couple rate: the 2024 rate ($1,415), the 2025 rate ($1,450), or projections between $1,482 and $1,490. The other misses zeroed the benefit through a false income or resource test, or gave an answer that contradicts their own arithmetic."
-us,scenario_079,state_refundable_credits,39,llm_error,state_local_rule,False,,"Arizona's increased excise tax credit is a refundable credit of $25 per filer, spouse and dependent, capped at $100, for tax units whose federal AGI is at or below $12,500 (MFJ/HOH) or $6,250 (single/MFS). It needs no earnings, children, tax liability or taxable income. This couple's only income is $7,560 of SSDI, which is below the federal Social Security taxation base, so federal AGI is $0 and head plus spouse qualify for 2 x $25 = $50. Every model that answered $0 tied Arizona's refundable credits to earnings, children, liability or property-tax/age tests, or wrongly called the excise credit nonrefundable, and so never counted this credit. The one model that did find the $50 credit then submitted $200 anyway."
+us,scenario_079,ssi,34,llm_error,thresholds_rates,False,,"Both spouses are blind and disabled, so they form an eligible SSI couple. Their countable resources are $1,045 in the bank, because the home and one vehicle are excluded, which is under the $3,000 couple limit. The head's $630/month SSDI minus the $20 general exclusion leaves $610/month of countable income. The 2026 couple federal benefit rate is $1,491/month, so the benefit is $1,491 − $610 = $881/month, or $10,572/year. Most models got the structure right but used a stale or misprojected couple rate ($1,415, $1,450, $1,482–$1,490, or inflated values). The rest wrongly found the couple ineligible, refused to compute SSI, or made arithmetic or submission errors."
+us,scenario_079,state_refundable_credits,40,llm_error,categorical_eligibility,False,,"Arizona's Increased Excise Tax Credit (A.R.S. §43-1072.01) is refundable. It pays $25 for each personal and dependent exemption, up to $100, to filers whose federal AGI is at or below $25,000 (MFJ/HOH) or $12,500 (single/MFS). It needs no earned income, tax liability, children, or minimum AZ taxable income. This MFJ couple's only income is non-taxable SSDI, so their federal AGI is $0, they are under the cap, and the credit is 2 × $25 = $50. Almost every model treated zero income or zero liability as ruling out refundable AZ credits. Several also misstated the credit as nonrefundable or tied to EITC, earnings, or filing thresholds."
us,scenario_080,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,head_chip_eligible,2,parse_contract_failure,other,False,,"The policy step is simple. The 41-year-old head has about $940 of income, so she qualifies for Pennsylvania Medicaid in the ACA expansion adult category, and Medicaid eligibility rules out CHIP. The correct answer is 0. Neither miss is a substantive policy error. Sonnet's explanation reached 0, but its scored value field records Yes. GLM returned no usable answer for this key at all."
-us,scenario_080,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The trap is that a disabled 41-year-old holding $19,828 in bank assets invites the aged/blind/disabled Medicaid pathway with its SSI-linked $2,000 resource limit, when Pennsylvania's ACA adult expansion group is what governs: it covers any adult 19-64 at or below 138% FPL, is MAGI-based, applies no asset or resource test, and requires no disability or SSI status. Countable MAGI is $940 ($924 long-term capital gains plus $16 qualified dividends; the $3,600 financial assistance is not MAGI income), which is 0.06 x FPL against a 138% FPL limit, and the prompt's default status facts satisfy the age and immigration/5-year tests the engine applied. Five of the six models either imported a resource or categorical screen that the MAGI adult group forbids, or inverted the income test by reading the near-total absence of income as evidence against eligibility."
+us,scenario_080,head_medicaid_eligible,7,llm_error,categorical_eligibility,False,,"Pennsylvania expanded Medicaid, so a 41-year-old non-dependent qualifies through the ACA adult group if MAGI is at or below 138% FPL. That group has no asset test and no disability or SSI requirement. The head's MAGI is only the $924 of long-term capital gains plus $16 of qualified dividends, about $940 or 0.06x FPL. The $3,600 of financial assistance is not taxable and is not counted in MAGI. Most models never applied this pathway: they demanded a disability or SSI category, applied an SSI-style $2,000 resource limit, or treated the household's low income as a reason to deny eligibility."
us,scenario_080,head_medicare_eligible,7,llm_error,categorical_eligibility,False,,"Medicare eligibility below age 65 requires a qualifying pathway such as Medicare entitlement after the required period of SSDI receipt, ALS, or end-stage renal disease; a generic disability flag alone does not establish any of those conditions. The head is 41, and SSDI receipt, the waiting period, ALS, and end-stage renal disease are all unlisted and therefore false, so the correct eligibility output is 0."
us,scenario_080,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_080,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_080,snap,46,reference_engine_defect,categorical_eligibility,False,,"The reference depends on three steps, and each wrong model departed from at least one. First, Pennsylvania's broad-based categorical eligibility through its TANF non-cash program waives the SNAP asset test, so the $19,828 bank balance does not matter; 14 models answered $0 on assets. Second, the $2,000 energy subsidy triggers Pennsylvania's $857 heating standard utility allowance (SUA) under heat-and-eat. For a disabled household the excess shelter deduction is uncapped, so it wipes out the $92.33 left after the $209 standard deduction ($301.33 gross, from $300 assistance plus $1.33 dividends, with capital gains excluded); 13 models stopped at the standard deduction. Third, the frozen reference pays FY2026's $298/month for January–September and the engine's projected FY2027 $304.68/month for October–December, whereas the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 for all 12 months (USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze); of the 16 models that answered 12 times a monthly maximum, 11 used a stale maximum ($291–$293) and 5 used $298 for all 12 months, which matches the convention. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (categorical_eligibility). The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Reference is an engine defect; output excluded from scoring (7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04))."
+us,scenario_080,snap,47,reference_engine_defect,categorical_eligibility,False,,"To reach $3,596.04, three steps are needed. First, Pennsylvania's broad-based categorical eligibility (through the TANF non-cash program) waives the SNAP asset test, so the $19,828 bank balance does not matter. Second, the $2,000 energy subsidy triggers the heat-and-eat standard utility allowance of $857 a month. Together with the standard deduction, that allowance brings net income to $0 every month. Third, the benefit is the one-person maximum allotment: $298 a month for January–September (FY2026) and $304.68 a month for October–December (FY2027), not one flat monthly figure times 12. The largest group of wrong models applied the asset test and answered $0. A second group skipped the utility allowance and subtracted 30% of net income. The rest reached the maximum benefit but used the FY2025 allotment ($291–$293) or $298 for all 12 months. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (categorical_eligibility). The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Reference is an engine defect; output excluded from scoring (7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04))."
us,scenario_080,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to be aged 65 or older, blind, or disabled under the program's categorical determination. The engine input sets the sole person's SSI aged/blind/disabled status to false, so benefit-rate and countable-income calculations never produce an award."
us,scenario_080,state_income_tax_before_refundable_credits,8,llm_error,credit_phaseout,False,,"Pennsylvania first applies its 3.07% rate to the $940 of taxable dividends and gains, then applies the nonrefundable Tax Forgiveness credit in computing income tax before refundable credits. The household's eligibility income remains within the range for 100% forgiveness, so the credit eliminates the preliminary tax and yields $0."
us,scenario_080,state_refundable_credits,2,llm_error,state_local_rule,False,,"Pennsylvania Schedule SP tax forgiveness is a nonrefundable credit that reduces Pennsylvania income-tax liability; it is not included in state refundable credits. With no qualifying Pennsylvania refundable credit, the requested aggregate is $0."
us,scenario_080,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_081,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"For 2026, OBBBA makes the TCJA 10/12/22/24% brackets and the $0 personal exemption permanent and raises the SALT cap to $40,400. The prompt lists no state income tax withheld or paid. In the reference, SALT is $14,127.41, fully deductible: $5,643 of real estate tax plus $8,484.41, the engine's estimate of Massachusetts income tax withheld (5% of federal AGI above the $4,400 personal exemption). Adding $16,900 of mortgage interest gives $31,027.41 of itemized deductions, well above the $16,100 standard deduction. Taxable income is $143,059.47, and the tax is $17,966 + 24% × $37,359.47 = $26,932.27. The engine's own Massachusetts income tax for the household is $8,238.41; with it in SALT the output is $26,991.31. The household's Massachusetts output is itself excluded for an engine defect, and with its corrected value, $8,232.90, the output is $26,992.64. With no state income tax paid, SALT takes $853 of general sales tax and the output is $28,763.81. The wrong answers fall into five groups: applying the expired $10,000 SALT cap (about $27,923), putting no state income tax and no sales tax in SALT (about $28,969), assuming a TCJA sunset with personal exemptions and 15/25/28% brackets (mostly about $29,100 to $30,200), subtracting the $7,389 ESI premium from the $175,002 wages, and near misses that put their own Massachusetts income tax estimate in SALT, mostly about $8,230 to $8,390, where the reference has the withholding estimate. Four answers are within $1 of the corrected-liability value: GPT-5.6 Sol ($26,992.59), Claude Opus 5.5 ($26,992.66), and GPT-6 Astra and GPT-6.1 Sol ($26,993.29). Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
+us,scenario_081,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The deciding step is the SALT part of itemized deductions. For 2026, OBBBA sets the SALT cap at $40,400, so the filer deducts the full $5,643 property tax plus the Massachusetts income tax. PolicyEngine counts that state tax as estimated withholding of 5% × (AGI $174,088.12 − $4,400) = $8,484.41, not the $8,238.41 final MA liability. This gives itemized deductions of $31,027.41 and, after the $1.24 QBI deduction, taxable income of $143,059.47. OBBBA's permanent 2026 single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%) turn that into $26,932.27. Most wrong answers made one of these errors: applied the old $10,000 TCJA cap, left MA income tax out of SALT, assumed TCJA expired (personal exemption and 15/25/28% brackets), took the standard deduction, subtracted ESI premiums from wages, or used the MA liability instead of the withholding estimate (about $60 high). Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
us,scenario_081,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_081,payroll_tax,37,prompt_ambiguity,state_local_rule,False,,"The reference has three parts: employee Social Security of 6.2% × $175,002 = $10,850.12 (the wages are below the 2026 wage base), employee Medicare of 1.45% × $175,002 = $2,537.53 with no Additional Medicare Tax, and $805.01 of Massachusetts Paid Family and Medical Leave (PFML), 0.46% × $175,002. The employer remits PFML contributions and, under the Department of Family and Medical Leave's published 2026 rates, may deduct up to 0.46% of wages from the employee (all of the 0.18% family rate and 40% of the 0.70% medical rate) or deduct less or none. The reference counts the most Massachusetts's published 2026 rates let the employer deduct; the prompt does not say what this employer does, and counting only what the law requires of the employee gives federal tax alone, $13,387.65. Nine models matched the reference. Of the other 37, nine answered federal tax alone, saying Massachusetts has no employee payroll tax or none is mandatory, assuming PFML is not modeled, leaving it out without a reason, or not mentioning it. Thirteen more left PFML out and made another error, most often a total that does not match the model's own figures; others used an outdated Social Security wage base, a 2.9% Medicare rate, Additional Medicare Tax on wages under $200,000, or a FICA base cut to $167,613 by subtracting the $7,389 of employer-sponsored insurance premiums, which the engine does not subtract. Thirteen included PFML at other amounts, at stated rates from 0.24% to 0.604% or as estimates without a rate, two of them on the $167,613 base. One returned $0 with no reasoning, and one returned no value. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_081,payroll_tax,38,prompt_ambiguity,state_local_rule,False,,"Federal FICA is the same in every correct path. The $175,002 in wages is below the 2026 Social Security wage base, so the employee owes 6.2% Social Security ($10,850.12) and 1.45% Medicare ($2,537.53), with no Additional Medicare Tax. That totals $13,387.65. The reference adds the Massachusetts PFML employee contribution at the maximum share an employer is allowed to withhold: the full 0.18% family rate plus 40% of the 0.70% medical rate, which is 0.46% × $175,002 = $805.01, for $14,192.66. The question asks for 'mandatory employee state payroll taxes,' and Massachusetts lets the employer absorb some or all of the PFML share. That gives two readings: (1) count PFML at the maximum withholdable employee share, which is the engine's reading, or (2) leave it out because the employee is not required to pay it, which gives $13,387.65. The prompt does not settle this. Many models gave exactly $13,387.65 under reading (2). The other wrong answers come from separate mistakes: wrong PFML rates, cutting the FICA wage base by the employer-sponsored insurance (ESI) premiums, outdated wage bases, or submitting a number that doesn't match their own arithmetic. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_081,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_081,state_income_tax_before_refundable_credits,43,reference_engine_defect,taxable_income_or_deductions,False,,"The reference has two parts. Part B income is $175,058: $175,002 of wages plus the $56 of taxable interest. Three items come off it: the $2,000 cap on the Social Security/Medicare deduction, the $4,000 rent deduction (50% of $44,400, capped at $4,000), and the $4,400 single exemption. That leaves $164,658, which is taxed at 5% for $8,232.90. Part A adds 5% on the $110 of dividends ($5.51), because the short-term capital loss is not netted against it, for a total of $8,238.41. Most models left out the $2,000 payroll-tax deduction or the $4,000 rent deduction, or used the old $3,000 rent cap. Others netted the $1,080 short-term loss against ordinary or Part A income, subtracted federal itemized deductions or ESI premiums that Massachusetts does not allow, or added arbitrary final adjustments. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes. Reference is an engine defect; output excluded from scoring (M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21)."
+us,scenario_081,state_income_tax_before_refundable_credits,44,reference_engine_defect,taxable_income_or_deductions,False,,"Massachusetts Part B income is wages of $175,002 plus $56 of bank interest, for $175,058. Three Part B items come off that: the $2,000 maximum deduction for Social Security/Medicare withholding, the renter deduction (50% of rent, capped at $4,000 after 2023) and the $4,400 single personal exemption. That leaves $164,658, and 5% of it is $8,232.90. Part A adds about $5.51 of tax on roughly $110 of dividends, for $8,238.41. Most models dropped the $2,000 payroll-tax deduction, used the old $3,000 rent cap or skipped the rent deduction, or took deductions Massachusetts doesn't allow (federal itemized deductions, employee business expenses, or subtracting employer insurance premiums from wages). A smaller group got Part B right but left out the roughly $8 of tax on the interest and dividends. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes. Reference is an engine defect; output excluded from scoring (M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21)."
us,scenario_081,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_chip_eligible,3,llm_error,age_disability,False,,"The decisive rule in the engine trace is CHIP's age criterion: child1 is age 1 and falls outside the applicable CHIP age range. The alternative Medicaid pathway also returns ineligible, so neither route produces CHIP eligibility."
us,scenario_082,child1_early_head_start_eligible,6,llm_error,categorical_eligibility,False,,"Early Head Start eligibility is a conjunction: the child must be under 3 (or the household must contain a pregnant person) AND the household must pass the means test — income at or below the federal poverty guideline, with limited over-income slots up to 130% FPL, or a categorical pathway such as TANF/SSI receipt, foster-care placement, or homelessness. Every wrong model treated the birth-to-3 age band as sufficient and either skipped or explicitly discarded the income screen. This household's countable income of ~$90,132 is more than four times the two-person poverty guideline (~$21k, ~$28k at 130%), no listed fact establishes TANF, SSI, foster care, or homelessness, and there is no pregnancy, so both the categorical and income conditions evaluate False and eligibility is No."
us,scenario_082,child1_medicaid_eligible,6,llm_error,health_coverage,False,,"The trap is that New York's generous children's coverage above 154% FPL runs through Child Health Plus — the state's separate CHIP program — not through Medicaid: NY Medicaid covers a child age 1–5 only up to 154% FPL, and the 400–405% FPL figures the models cited are the CHIP ceiling. The second half of the trap is income: Medicaid MAGI here is about $117.8k (543% FPL for a two-person household) because the $33,000 long-term capital loss is limited to a $3,000 deduction against ordinary income and because $20,580 of qualified and non-qualified dividends, taxable interest, short-term gain, and the state tax refund all count, while the $21,208 ESI premium is not subtracted from MAGI. Every model either imported a CHIP-scale threshold into Medicaid, understated MAGI to roughly $90k–$99k, or asserted child eligibility with no income test at all. At 543% FPL the child clears no Medicaid category (medicaid_category = NONE), so is_medicaid_eligible is False."
us,scenario_082,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_wic_eligible,2,llm_error,thresholds_rates,False,,"WIC requires both categorical eligibility and financial eligibility; being age 1 satisfies only the categorical test. The household's approximately $90,132 annual income exceeds the 2026 WIC limit of 185% of the federal poverty guideline for a two-person household, so the child is not eligible."
-us,scenario_082,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference starts from AGI of $117,585.14. That is wages net of the $181 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capped capital loss and the $8 IRA deduction. The $68 state refund is excluded because the filer did not itemize. It then subtracts the $24,150 2026 head-of-household standard deduction and the new $700 non-itemizer charitable deduction, giving taxable income of $92,735.15. It taxes $79,311.15 of ordinary income on the 2026 HoH schedule (10% to $17,700, 12% to $67,450, 22% above) for $10,349.45, and the $13,424 of qualified dividends at 15% for $2,013.60. Finally it subtracts the $600 CDCC and the $2,200 CTC. The closest answers ($9,578–$9,734) differ only by skipping the $700 non-itemizer charitable deduction and/or adding the $68 refund. The rest used stale brackets or standard deductions, a $2,000 CTC, TCJA-sunset rules (personal exemptions, 10/15/25 brackets, $1,000 CTC), an uncapped capital loss, or deducted the $21,208 employer-sponsored premiums. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 9,563.05; the corrected value is 9,564.92. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_082,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The 2026 computation follows post-OBBBA law. AGI is $117,661: the net capital loss is capped at $3,000, the SALT refund is included, and the $8 traditional IRA deduction is zero because the head is an active 401(k) participant above the phase-out range. Taxable income takes off the $24,150 HoH standard deduction plus the new $700 charitable deduction for non-itemizers. The ordinary portion is taxed at the 2026 HoH brackets (10% to $17,700, 12% to $67,450, then 22%), and the $13,424 of qualified dividends at 15%. Then the $600 CDCC and the $2,200 CTC are subtracted. The closest misses all dropped the $700 non-itemizer charitable deduction (a $154 overstatement). Others used the $2,000 CTC, pre-OBBBA brackets or standard deduction, TCJA-sunset rules with personal exemptions, itemized deductions that counted employer-sponsored insurance premiums as medical expenses, or netted the full $30,711 capital loss."
us,scenario_082,federal_refundable_credits,8,llm_error,credit_phaseout,False,,"The separating step is that the Additional Child Tax Credit is a residual, not a per-child entitlement: the 2026 CTC of $2,200 is first applied nonrefundably against tax liability, and only the unused remainder — capped at $1,700 and further limited to 15% of earned income over $2,500 — becomes refundable. This head-of-household filer has roughly $117,650 of AGI (wages net of the $181 traditional 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capital-loss limitation on a $30,711 net loss and the $8 IRA deduction), which produces federal tax before credits many times larger than $2,200, so the entire credit is consumed nonrefundably and nothing spills into the refundable bucket. EITC is zero on income grounds and independently on the investment-income limit, and no American Opportunity Credit, Recovery Rebate Credit, or refundable payroll tax credit applies. Every wrong model quoted a statutory refundable ceiling as the answer instead of computing the leftover after the nonrefundable CTC."
us,scenario_082,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_082,local_income_tax,5,llm_error,state_local_rule,False,,"New York State residence does not establish New York City residence, and the prompt supplies no city or locality. Because unlisted location facts are false, no covered local income, wage, or earnings tax applies, so the separate local-income-tax output is zero."
-us,scenario_082,payroll_tax,42,prompt_ambiguity,state_local_rule,False,,"The reference is $8,108.03: employee Social Security and Medicare, 7.65% of the full $100,195 of wages ($6,212.09 plus $1,452.83, or $7,664.92), plus $443.11 of New York contributions: Paid Family Leave (PFL) at 0.432% of wages up to its 2026 annual maximum of $411.91, and disability benefits (DBL) at $0.60 a week, $31.20 a year. New York lets the employer collect these from the employee up to those caps but does not require it, and the employer pays the disability cost it does not collect. The reference counts the most the employer may collect; the prompt does not say what this employer does, and counting only what the law requires of the employee gives federal tax alone, $7,664.92. Four models matched the reference. Of the other 42, fourteen answered federal tax alone (about $7,665), saying New York has no employee payroll tax or none is modeled, calling the contributions negligible, or not mentioning them. Eight more left both out and made another error: FICA understated or mis-added (answers from $7,654 to $7,765 instead of $7,664.92), or a FICA base cut by subtracting the $21,208 employer-sponsored insurance premium or the $181 traditional 401(k) deferral, which the engine does not subtract. Eighteen included PFL or DBL at other amounts: PFL at rates, caps or estimates other than 0.432% up to $411.91, or at 0.432% with no cap; DBL alone; or PFL without DBL; four of them also cut the FICA base by the insurance premium. One subtracted an invented $200 of New York deductions from FICA, and one returned no value. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_082,local_income_tax,6,llm_error,state_local_rule,False,,"The household facts give only state = NY and never say the household lives in New York City. The prompt says to treat any unlisted status input as false, so NYC residence is false. In New York, only NYC residence triggers the separate local income tax output, so local_income_tax is $0. Every wrong model assumed NYC residence and ran NYC resident brackets on NY taxable income, which turned a state-only fact pattern into an NYC tax bill."
+us,scenario_082,payroll_tax,43,prompt_ambiguity,state_local_rule,False,,"The answers split on whether New York's employee Paid Family Leave and disability (DBL) contributions count. The reference adds both at their statutory maximums: PFL at 0.432% of $100,195, capped at the 2026 maximum of $411.91, and DBL at $0.60 a week, or $31.20 a year. That puts $443.11 on top of FICA of $7,664.92. New York lets employers collect less or none, so the prompt's phrase ""mandatory employee state payroll taxes"" supports two readings: (1) the statutory maximum employee PFL/DBL contributions count, or (2) they are not mandatory employee taxes, which leaves FICA only. Eighteen models took reading (2). The rest included New York items but used old PFL rates or caps (0.373% or 0.388%, maximums of $333–$370), skipped the $411.91 cap, left out PFL or DBL, or cut FICA wages by subtracting the ESI premiums or the 401(k) deferral. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_082,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_082,state_income_tax_before_refundable_credits,45,llm_error,state_local_rule,False,,"The filer is a New York head of household. NY AGI is $117,585. That is federal AGI less the $68 refund subtraction, with the capital loss capped at $3,000 and the listed wages not reduced by the ESI premiums. Subtracting the $11,200 HOH standard deduction (which beats the $11,169 of NY itemized deductions) and a $1,000 exemption for the child only leaves $105,385.15. The 2026 reduced HOH rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17 on that amount, and a $156.38 supplemental recapture tax applies because NY AGI exceeds $107,650. No nonrefundable credit applies: the NY child and dependent care credit and the Empire State Child Credit are refundable, and the household credit is phased out. The models went wrong in several ways: filing as single or using the $8,000 deduction, adding an exemption for the head, using pre-2026 rates, leaving out the supplemental tax, subtracting refundable credits, over-itemizing, or removing the ESI premiums or the uncapped capital loss from income."
-us,scenario_082,state_refundable_credits,46,llm_error,state_local_rule,False,,"This household gets two refundable New York credits. The first is the enhanced 2026 Empire State Child Credit: $1,000 for a child under 4, reduced by $16.50 for each whole $1,000 of federal AGI ($117,652.65) above the $75,000 head-of-household threshold. That is 42 steps, or $693, which leaves $307. The second is the refundable NY child and dependent care credit at 60% of the $600 federal credit, or $360, for a total of $667. The wrong answers fall into four patterns. Some applied the pre-2025 ESCC (33% of the federal CTC, a $100 minimum, or no credit for children under 4). Some treated the ESCC as fully phased out. Some used the 20% NY CDCC percentage instead of 60% or called the CDCC nonrefundable. Others ignored one of the two credits entirely."
+us,scenario_082,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Getting the reference takes four New York-specific steps. First, find NY AGI: subtract the $68 SALT refund from federal AGI. Federal AGI caps the capital loss at $3,000 and doesn't include the ESI premiums. Second, use head-of-household status, which gives an $11,200 standard deduction (more than the $9,328 of NY itemized deductions) plus a $1,000 dependent exemption, for $105,393.62 of taxable income. Third, apply the reduced 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) to get $5,442.63, then add $156.51 of supplemental recapture tax because NY AGI exceeds $107,650. Fourth, subtract no nonrefundable credits: the NY child-care credit and the Empire State Child Credit are refundable, and the household credit is phased out at this income. The models went wrong on filing status, the dependent exemption, the capital-loss cap, the ESI premiums, the pre-2026 rates, the omitted supplemental tax, or by treating the refundable NY child-care credit as nonrefundable."
+us,scenario_082,state_refundable_credits,47,llm_error,state_local_rule,False,,"Two NY rules make up the $1,187.55. First, the enhanced Empire State Child Credit gives $1,000 for a child under 4. It is cut by $16.50 for each full $1,000 of federal AGI above the $75,000 head-of-household threshold: 42 full increments = $693, leaving $307. Second, the 2026 child care credit under Tax Law 606(c-2) is no longer a percentage of the federal CDCC. It applies an AGI-based percentage (about 29.352% at NY AGI of $117,594) directly to the $3,000 of qualifying expenses, giving $880.55 of refundable credit. Nearly every model used the old rule (20% of the $600 federal credit, or $120), said the care credit was nonrefundable, or used the pre-2025 ESCC rules: the $100/33% formula, under-4 children excluded, or an unphased $1,000."
us,scenario_083,federal_income_tax_before_refundable_credits,1,llm_error,thresholds_rates,False,,"The 2026 single-filer standard deduction exceeds the household’s $11,010 AGI, leaving zero taxable income. With no other federal tax components or nonrefundable credits affecting the result, federal income tax before refundable credits is $0."
us,scenario_083,federal_refundable_credits,9,llm_error,categorical_eligibility,False,,"A childless filer must satisfy the EITC age requirement, and the 20-year-old head does not. With no qualifying children and no applicable refundable education, recovery rebate, or payroll tax credit, every component of federal refundable credits is zero."
us,scenario_083,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Texas did not adopt ACA Medicaid expansion, so income below 138% of FPL does not create Medicaid eligibility for a nondependent adult. The 20-year-old head has no qualifying Texas category—such as pregnancy, parent/caretaker status, disability, or SSI eligibility—so `medicaid_category` is NONE regardless of the 0.69 FPL MAGI level."
us,scenario_083,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 20-year-old adult passes the income and nutritional-risk tests but belongs to category 5, so categorical ineligibility makes head_wic_eligible equal to No."
-us,scenario_083,payroll_tax,5,llm_error,payroll_tax_base,False,,"Payroll tax applies the 6.2% Social Security rate and 1.45% Medicare rate to the $7,050 wage base, with no Additional Medicare Tax or Texas employee payroll tax. The components are $437.10 and $102.22, producing $539.33 after rounding; rental income and the separately listed overtime premium do not enter this wage base."
+us,scenario_083,payroll_tax,6,llm_error,other,False,,"The correct answer is the employee FICA rate applied once to the $7,050 of gross wages, which already include overtime pay: 6.2% Social Security ($437.10) plus 1.45% Medicare ($102.225), for $539.33. Rental income is not payroll wages, the FLSA overtime premium is part of the $7,050 and is not added again, wages are far below the Additional Medicare Tax threshold, and Texas has no employee state payroll tax. Four models used the right base and rates but made arithmetic slips or applied made-up rounding to their final number. Two used too large a wage base: one added the overtime premium a second time, and one gave a number implying about $8,492 of wages."
us,scenario_083,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Ordinary rental income is generally excluded from net earnings from self-employment and therefore does not enter the self-employment tax base. With no reported self-employment earnings, the $400 filing threshold is not met and self-employment tax is zero."
us,scenario_083,snap,18,llm_error,asset_resource,False,,"This one-person Texas household passes every SNAP income test: gross income is $917.50 a month, well under 130% and 165% of FPL, and net income is about $591, under 100%. What makes it ineligible is its resources. Texas's broad-based categorical eligibility keeps a $5,000 asset limit, and the federal standard for a non-elderly household is about $3,000. The head's $6,620 bank balance exceeds both, so SNAP is $0. Every model ran the income-based benefit formula, getting about $110-$245 a month. None applied the resource test, and several said outright that Texas BBCE waives the asset test."
us,scenario_084,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Disability, blindness, SSDI receipt, and qualifying immigration status do not independently establish Medicaid eligibility; the person must satisfy a specific North Carolina coverage pathway and its financial rules. The head qualifies through no Medicaid category, and Medicaid MAGI is 1.56 times FPL, above the expansion threshold."
-us,scenario_084,head_medicare_eligible,33,prompt_ambiguity,age_disability,False,,"The Head is 62, which is under the Medicare age threshold of 65, and has no end-stage renal disease. That leaves the disability pathway, which requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)). PolicyEngine tests this with a count of months receiving SSDI. That count is not listed, so under the prompt's rule it is 0. The prompt's instruction to hold status constant covers only the 12 months of the tax year, not the earlier history needed for 24 months. Every model treated being disabled plus current SSDI income as enough for Medicare. Many said the 24-month wait was met even though no listed fact supports that. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
+us,scenario_084,head_medicare_eligible,34,prompt_ambiguity,age_disability,False,,"The head is 62, so they don't meet Medicare's age-65 test. Under 65, PolicyEngine grants Medicare eligibility only after at least 24 months of Social Security disability receipt, or for ESRD. The household facts never give how many months the head has received SSDI. The prompt says unlisted numeric inputs are 0, so the months of SSDI receipt are 0 and the 24-month waiting period is not met. Every model treated being disabled plus current SSDI income as Medicare eligibility, or assumed the 24 months were already served. That assumption broke the prompt's explicit rule against inferring unlisted facts. Even counting all 12 months of SSDI in the tax year would still fall short of 24. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_084,snap,2,llm_error,thresholds_rates,False,,"The single-person household has $33,233 of gross annual income ($2,769/month), which exceeds North Carolina's broad-based categorical eligibility gross cap of 200% FPL (about $2,680/month for one person in 2026), and after the ~$204 standard deduction and the elderly/disabled excess-medical deduction its net income of roughly $1,738/month still exceeds the 100%-FPL net income limit of about $1,340/month; the $56,500 bank balance also exceeds the elderly/disabled resource limit of roughly $4,650. Ineligibility ends the calculation at $0. Both models stopped at the observation that the gross income test is waived when an elderly or disabled member is present and never tested net income against the 100%-FPL ceiling, then manufactured a benefit — one by overriding its own $0 formula result, the other by applying the one-to-two-person minimum allotment, which is a floor for households already found eligible, not a substitute for eligibility."
-us,scenario_085,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"AGI is $45,573.60: $31,523 of other income plus taxable Social Security of $14,050.60, which is 85% of the $11,236 by which provisional income exceeds $34,000, plus $4,500. The step that separates the reference from most wrong answers is the 2026 deduction stack under OBBBA. It combines the $16,100 standard deduction, the $2,050 additional amount for age 65+, and the $6,000 senior deduction (2025-2028, no phase-out below $75,000 MAGI), for $24,150 in total. That leaves taxable income of $21,423, taxed at 10% up to $12,400 and 12% above, for $2,322.76. Models that left out the $6,000 senior deduction landed near $3,040-$3,190. Models that assumed the TCJA would sunset, bringing back personal exemptions and a 15% bracket, landed near $3,800-$4,200. The rest overstated taxable Social Security, invented nonrefundable credits, or submitted a value their own derivation did not support."
+us,scenario_085,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"The correct path starts with taxable Social Security of about $14,051, which is $4,500 plus 85% of provisional income above $34,000. That gives AGI of about $45,574. Under OBBBA, the TCJA structure is permanent, so 2026 has no personal exemption and a $16,100 single standard deduction plus $2,050 for being 65 or older, for $18,150. The $6,000 OBBBA senior deduction also applies because MAGI is under $75,000, bringing total deductions to $24,150. Taxable income is $21,423.74, taxed at 10% up to $12,400 and 12% above that, for $2,322.85, and there are no nonrefundable credits. Most models either left out the $6,000 senior deduction, assumed TCJA had expired (personal exemption and a 15% bracket), miscalculated taxable Social Security, or submitted a number that didn't match their own correct work."
us,scenario_085,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_085,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_085,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not Social Security survivor benefits, retirement distributions, pension income, or taxable interest. Because the household has no listed wages or other payroll-taxable earnings, the employee-side payroll-tax base is zero and payroll tax is $0."
@@ -674,12 +673,12 @@ us,scenario_085,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_085,state_income_tax_before_refundable_credits,34,llm_error,state_local_rule,False,,"This 67-year-old's Social Security and retirement distributions are exempt from PA tax, so PA taxable income is just the $1,675 of interest. At 3.07% that is $51.42 of tax. Pennsylvania's Special Tax Forgiveness (Schedule SP) then removes it entirely. Eligibility income for forgiveness leaves out Social Security and commonly recognized retirement benefits, so it is only $1,675, well under the $6,500 single-filer limit for 100% forgiveness. Most models stopped at $51.42, either never applying forgiveness or rejecting it because they counted the $27,426 of Social Security and $29,848 of retirement income as eligibility income. A few taxed the retirement distributions outright."
us,scenario_086,child1_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires Medicaid ineligibility and satisfaction of CHIP’s independent age and income criteria. Child 1 is age 12, so age is not the failing criterion; the household income exceeds Georgia’s applicable CHIP income threshold for 2026."
us,scenario_086,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The models conflated Georgia’s separate PeachCare/CHIP pathway with Medicaid or assumed that being a child creates Medicaid eligibility at higher income levels. At 3.44 times FPL, child1 exceeds Georgia’s child Medicaid thresholds and qualifies for no Medicaid category, so employer-sponsored coverage and non-MAGI expense deductions do not create eligibility."
-us,scenario_086,federal_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The Social Security dependent benefits are the Head's input. Provisional income ($61,662.50 + $6,412 = $68,074.50) exceeds the $34,000 single/HoH second base, so the 85% cap applies and $10,900.40 is taxable, which puts AGI at $72,562.90 after the full $337.50 educator deduction. Under OBBBA's permanent TCJA structure, the 2026 HoH standard deduction is $24,150, which leaves taxable income of $48,412.90. Tax is $1,770 + 12% of $30,712.90 = $5,455.55, and the full $2,200 CTC is absorbed as a nonrefundable credit, giving $3,255.55. The wrong answers split into four groups. Some dropped the taxable SS as the child's income. Some applied TCJA-sunset rules (personal exemptions, 15% bracket, $1,000 CTC). Some used stale 2025 or pre-OBBBA parameters ($22,500/$23,100 SD, $17,000 bracket, $2,000 CTC). The rest used the wrong filing status or took an unsupported auto-loan-interest deduction or $300 educator cap. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)."
+us,scenario_086,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,"PolicyEngine allows a $337.50 above-the-line educator expense deduction for 2026. IRC §62(a)(2)(D) and §62(d)(3) index the $250 base and round it down to a multiple of $50, which gives a 2026 limit of $300. The engine appears to uprate the cap without that round-down, which overstates the deduction by $37.50 and understates tax by $4.50. The statutory value would be $3,260.05.","The $12,824 of Social Security dependent benefits is listed on Head, so it counts as Head's income. Provisional income is about $68,075, well above the $34,000 HoH adjusted base, so 85% ($10,900.40) is taxable and AGI is $72,562.90. In 2026 the OBBBA-extended TCJA rules apply: a $24,150 HoH standard deduction, a 10% bracket up to $17,700 and a 12% bracket above it, no personal exemptions, and a $2,200 CTC used in full against $5,455.55 of tax. The household facts do not support a car-loan interest deduction. Most wrong answers left out the taxable Social Security, used TCJA-sunset rules (exemptions, 15% bracket, $1,000 CTC), used an older standard deduction or bracket threshold, or deducted the $275 auto-loan interest. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)."
us,scenario_086,federal_refundable_credits,11,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is a residual, not an entitlement: IRC 24(d)(1) makes it the lesser of the per-child refundable cap ($1,700 for 2026) and the amount of CTC that the §26(a) tax-liability limitation actually blocked, which Schedule 8812 computes as line 12 minus line 14 (""If zero, stop here; you cannot take the additional child tax credit""). This head-of-household filer has AGI of $72,562.90 ($62,000 wages plus $10,900.40 of the $12,824 Social Security dependent benefit that is taxable once combined income exceeds the $34,000 tier-2 threshold, less the $337.50 educator deduction), taxable income of $48,412.90 after the 2026 HoH standard deduction of $24,150, and pre-credit tax of $5,455.55 — so the full $2,200 CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is $0 because earned income of $62,000 is far past the 2026 one-child HoH phase-out endpoint of roughly $51,600, computed at the statutory 15.98% phase-out rate, and no education expenses, rebate, or refundable payroll credit exist here. Ten of eleven models treated the $1,700 refundable cap (or a stale $1,000/$1,500 figure) as a floor paid out per qualifying child and never performed the unused-CTC subtraction against liability."
us,scenario_086,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_086,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_086,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee payroll tax is $3,844 of Social Security tax plus $899 of Medicare tax on the head’s full $62,000 wage amount, totaling $4,743. The listed insurance premiums do not reduce the FICA wage base in this computation, Georgia unemployment insurance is employer-paid rather than a mandatory employee payroll tax, and the dependent child has no wages."
-us,scenario_086,state_income_tax_before_refundable_credits,45,llm_error,thresholds_rates,False,,"Georgia 2026 tax starts from federal AGI ($72,562.90), subtracts the $10,900.40 of taxable Social Security to reach GA AGI of $61,662.50, then subtracts the $15,000 head-of-household standard deduction and a $5,000 dependent exemption. There is no taxpayer personal exemption. The resulting $41,662.50 is taxed at the flat 2026 rate of 4.99%, giving $2,078.96. Almost every model used out-of-date parameters: a $12,000 (or $18,500 or $24,000) standard deduction, a $3,000–$4,000 dependent exemption, a flat rate between 5.09% and 5.49%, or the repealed pre-2024 structure of a $5,400 standard deduction plus $2,700 personal exemptions. A smaller group also left Social Security in Georgia income or took deductions that don't exist (employer insurance premiums, auto-loan interest, or the educator expense counted twice)."
+us,scenario_086,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Georgia AGI is federal AGI ($72,562.90, which includes $10,900.40 of taxable Social Security) minus the full taxable Social Security subtraction, giving $61,662.50. For 2026, PolicyEngine then applies Georgia's $15,000 head-of-household standard deduction and a $5,000 dependent exemption, with no separate personal exemption. That leaves taxable income of $41,662.50, taxed at the flat 4.99% rate, for $2,078.96. Nearly every model kept older Georgia parameters: a $12,000/$18,500/$24,000 standard deduction, $3,000/$4,000 dependent exemptions, pre-2024 $2,700 personal exemptions, or a 5.09–5.49% rate. Several also mishandled the Social Security subtraction or employer health premiums."
us,scenario_086,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"Georgia does not provide the claimed $260 refundable credit for each dependent child under age 17. With no applicable Georgia refundable credit, the state-level refundable-credit aggregate is $0."
us,scenario_088,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The decisive computation step is the 2026 deduction for qualified tip income: the $9,600 of tips included in wages reduces taxable income in addition to the applicable standard deduction. The state income-tax refund is also excluded because no prior-year itemized-deduction tax benefit is established. These deductions reduce taxable income to zero, so federal income tax before refundable credits is $0 and cannot be negative."
us,scenario_088,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The only refundable credit in play is the childless EITC, and IRC §32(a)(2)(B) phases it out against the GREATER of earned income or AGI. Earned income is $18,577, but AGI is $24,035 once the $2,733 unemployment compensation, $1,862 taxable pension, $675 state tax refund, $140 interest and $48 dividends are added; the tip deduction is below-the-line and does not reduce AGI. The 2026 childless EITC completes its phase-out below $20,000, so an AGI of $24,035 zeroes the credit, and with no qualifying children, no education expenses and no rebate, every other refundable component is $0. Every wrong model ran the phase-out on earned income alone, or on a max-credit shortcut, and never applied the AGI leg of the test."
@@ -687,11 +686,11 @@ us,scenario_088,head_medicaid_eligible,5,llm_error,categorical_eligibility,False
us,scenario_088,head_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility for a person under 65 is not conferred by disability status itself: it requires 24 months of entitlement to Social Security Disability Insurance (or RRB disability benefits), or an ESRD/ALS diagnosis. The head is 56 with $0 Social Security disability income — every unlisted input is 0, so there is no SSDI entitlement and no 24-month qualifying period — and the only qualifying pathway left, age 65, is unmet, so the answer is No. All six models converted the household's `is_disabled` (and `is_blind`) boolean directly into Medicare entitlement, skipping the SSDI-receipt precondition that actually gates the under-65 pathway; blindness in particular is an SSI/SSDI disability criterion, not a Medicare one."
us,scenario_088,payroll_tax,5,llm_error,payroll_tax_base,False,,"The employee payroll tax base here is the full $18,577 of W-2 wages, tips included: IRC 3121(a) and 3121(q) make reported tips wages for both Social Security and Medicare, and the tip deduction is an income-tax deduction that leaves the FICA base untouched. Everything else in the household — the $1,862 taxable pension, $2,733 unemployment compensation, $140 interest, $48 dividends — sits outside the payroll tax base, wages fall far below both the Social Security wage base and the $200,000 Additional Medicare Tax threshold, and Texas imposes no mandatory employee payroll tax. The entire computation is 0.062 × 18,577 = $1,151.78 plus 0.0145 × 18,577 = $269.37, totaling $1,421.14. The wrong answers come from zeroing the wage base entirely, carving tips out of Social Security wages while inventing an employee tip credit, or misexecuting the 7.65% multiplication."
us,scenario_088,snap,5,llm_error,asset_resource,False,,"The head holds $31,776 in bank account assets, and that fails the SNAP resource test outright. The balance is far above the federal $4,500 limit for households with an elderly or disabled member and also above Texas's $5,000 resource limit under broad-based categorical eligibility. Because the household fails the asset test, SNAP is $0 whatever its net income or shelter deduction works out to. Every wrong model took the disability-based waiver of the gross income test and went straight to the net-income and shelter-deduction calculation. None of them applied the resource test to the listed bank balance."
-us,scenario_089,federal_income_tax_before_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"AGI is $106,200.13. That is the spouse's wages net of only the spouse's own $6,946 traditional 401(k), plus $30,335 of self-employment income and $19,650 of partnership/S-corp income, minus half of SE tax ($2,143) and both traditional IRAs ($577). The employer-sponsored insurance premiums are not excluded from wages, and the head's 401(k) has no wages to reduce. Taxable income then subtracts three below-the-line deductions under 2026 OBBBA law: the $32,200 MFJ standard deduction, a QBI deduction of 20% of $47,842.18 (QBI net of the half-SE-tax deduction), and the new qualified overtime deduction for the spouse's $3,467 FLSA overtime premium. That leaves $60,964.27 taxed at 10% up to $24,800 and 12% above. Most models missed the overtime deduction. Others assumed the TCJA sunset (personal exemptions, 15% bracket), subtracted the ESI premiums from wages, deducted the head's 401(k), or skipped the QBI deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The correct 2026 chain is: AGI $106,200.13, made up of the spouse's wages net of only their own $6,946 401(k), plus SE income and partnership income, minus half of SE tax and both IRAs. ESI premiums are not excluded from wages, the head's 401(k) is not deducted, and there is no SE health insurance deduction. From AGI, subtract the OBBBA $32,200 MFJ standard deduction, a QBI deduction of 20% × $47,842.18 (SE QBI is reduced by the half-SE-tax deduction), and the new $3,467 qualified overtime deduction. That leaves $60,964.27 taxable, taxed at 10% up to $24,800 and 12% above. The trap that caught the most models was missing the OBBBA overtime deduction. Other common errors were assuming the TCJA sunset (personal exemptions and a 15% bracket), excluding the $7,789 ESI premium from wages, deducting the head's 401(k) even though the head has no wages, and skipping or overstating the QBI deduction."
us,scenario_089,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Refundable CTC requires a qualifying child, and this two-adult household has none. Excess Additional Medicare Tax withholding is not a refundable tax credit, and the model also invented wages for the head despite the instruction that unlisted numeric inputs are zero; therefore all enumerated refundable-credit components equal zero."
-us,scenario_089,payroll_tax,21,llm_error,payroll_tax_base,False,,"Only the spouse's $65,881 of gross wages counts toward employee payroll tax. The prompt says gross wages already include overtime, so the $3,467 FLSA overtime premium is not added again. The facts list no pre-tax salary-reduction election, so the $7,789 employer-sponsored insurance premium, the traditional 401(k) deferrals, and the individually paid health premiums do not reduce the wage base. The result is 6.2% × $65,881 = $4,084.63 plus 1.45% × $65,881 = $955.28, for $5,039.91. The head has no wages, self-employment and partnership income are outside employee FICA, and North Carolina has no mandatory employee state payroll tax. The most common error, made by 9 models, was subtracting the $7,789 ESI premium to get $58,092 × 7.65% = $4,444. The other models double-counted overtime, invented wages for the head, subtracted 401(k) or other premiums, added self-employment tax, or made arithmetic slips."
-us,scenario_089,self_employment_tax,12,llm_error,payroll_tax_base,False,,"The self-employment tax base here is the spouse's $30,335 self-employment income and nothing else: the $19,650 of partnership/S-corp income flows to taxable income but stays outside net earnings from self-employment, and the household's $69,348 of wages are far below the Social Security wage cap, so no wage-base coordination reduces the SECA Social Security component. Taxing 92.35% of $30,335 at the full 15.3% (12.4% Social Security + 2.9% Medicare) produces the $4,286.24 reference. Five models inflated the base to $49,985 by adding the partnership/S-corp income, landing near $7,063; the rest chose the right base but broke on execution — mis-multiplying the 92.35% factor, dropping the factor entirely, or submitting a final number their own stated arithmetic never produced."
-us,scenario_089,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"North Carolina taxes federal AGI minus the $25,500 married-filing-jointly standard deduction at the 2026 flat rate of 3.99%. Federal AGI is $106,200.13. It starts with the spouse's $65,881 in wages, which already include overtime, minus only the spouse's own $6,946 traditional 401(k) deferral. It adds the full $30,335 of self-employment income and the $19,650 partnership/S-corp income, then subtracts the roughly $2,143 half-of-self-employment-tax deduction and both spouses' $577 in IRA deductions. The largest group of wrong answers ($2,909) removed the spouse's $7,789 employer-sponsored insurance premium from wages. Other models deducted Head's 401(k) even though Head has no wages, disallowed Head's spousal IRA deduction, used the 0.9235 self-employment tax factor in AGI, subtracted the overtime deduction (which is taken after AGI), or used old 4.25%/4.5% rates or wrong standard deductions."
+us,scenario_089,payroll_tax,22,llm_error,payroll_tax_base,False,,"The only wage earner is the spouse, whose FICA base is the full $65,881 of gross wages. The prompt says that figure already includes overtime, so the $3,467 FLSA premium is not added again. The $7,789 employer-sponsored insurance premium, traditional 401(k) deferrals and other health premiums are not subtracted. The head has no wages, NC has no mandatory employee payroll tax, and self-employment and partnership income are excluded. That leaves 6.2% + 1.45% of $65,881, which is about $5,039.91. Most models made one of three mistakes: they subtracted the ESI premium (giving $4,444), added the overtime premium on top of gross wages (giving a base of $69,348), or brought in wages for the head or non-wage income that the prompt does not list."
+us,scenario_089,self_employment_tax,13,llm_error,payroll_tax_base,False,,"In PolicyEngine, the self-employment tax base is only the spouse's $30,335 of self-employment income. The calculation is $30,335 × 0.9235 = $28,014 in net earnings, times 15.3%, which gives about $4,286. The SE base is far below the 2026 Social Security wage base even after adding the spouse's $65,881 in wages, so the full 12.4% applies. The $19,650 of partnership/S-corp income is not part of the SE tax base. Five models added it to the base and got about $6,400–$7,100. Most of the others used the right method but made arithmetic or rounding errors, or gave a final number that doesn't follow from their own calculation."
+us,scenario_089,state_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"North Carolina tax is a flat 3.99% (2026) of federal AGI minus the $25,500 MFJ standard deduction. Federal AGI is $106,200: spouse wages of $65,881 less only the spouse's $6,946 traditional 401(k), plus $19,650 pass-through income and $30,335 SE income, less half of SE tax (~$2,143) and $577 of IRA deductions. The head's IRA counts through the spousal-IRA rule, but the head's 401(k) does not because the head has no wages. That gives taxable income of $80,700 × 3.99% = $3,219.94. Most wrong answers came from three mistakes. Some models subtracted the $7,789 ESI premium input from wages, giving an AGI of $98,411. Others subtracted the head's $5,403 401(k) even though the head has no wages. The rest used an outdated 4.25% or 4.5% rate or a wrong standard deduction."
us,scenario_089,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_090,federal_refundable_credits,10,llm_error,age_disability,False,,"For tax year 2026, a filer with no qualifying children must be at least age 25 to claim the childless EITC. The head is 22, so the EITC is $0 regardless of the 7.65% phase-in calculation; with every other refundable-credit component also equal to $0, federal refundable credits total $0."
us,scenario_090,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Kansas has no Medicaid expansion pathway for a nondisabled, nonpregnant, childless 22-year-old adult. A low MAGI of 0.44 times FPL does not independently establish eligibility: the person must first belong to a covered Medicaid category, and this head belongs to none."
@@ -701,12 +700,12 @@ us,scenario_090,snap,8,llm_error,asset_resource,False,,"Kansas does not use broa
us,scenario_090,state_income_tax_before_refundable_credits,20,llm_error,taxable_income_or_deductions,False,,"Kansas's 2024 tax law (SB 1) replaced the old $2,250 per-person exemption with a $9,160 personal exemption for a single filer, on top of the $3,605 single standard deduction. Together they shelter $12,765, which is more than this filer's $7,020 Kansas AGI, so Kansas taxable income is $0 and so is the tax. Every model used the pre-2024 exemption ($2,250), the dependent-sized $2,320 figure, a $110 credit, or no exemption at all, which left $900 to $3,520 of phantom taxable income. Several models then made it worse with retired rates (3.1%, 5.15%, 5.5%, 5.7%), but the exemption error alone produces the nonzero answer."
us,scenario_090,state_refundable_credits,5,llm_error,categorical_eligibility,False,,"Kansas's refundable individual credit is its EITC, set at 17% of the federal EITC, and the head is 22 — below the age-25 floor for the childless federal EITC — so the federal credit is $0 and the state match on it is $0. The other candidate, the Kansas food sales tax credit, is nonrefundable for tax years after 2012 and in any case requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 in the home, none of which describes a 22-year-old single filer with no dependents. Every wrong answer cleared a household income test ($7,020 is well under both credits' ceilings) and then stopped, never applying the categorical age/dependent gate that actually decides both credits. The correct derivation is 17% × $0 federal EITC plus $0 qualifying food sales tax credit = $0."
us,scenario_090,tanf,1,llm_error,categorical_eligibility,False,,"TANF is a benefit for families with dependent children; a single 22-year-old with no children fails Kansas's categorical requirement outright, and the $20,300 in bank assets independently exceeds the state's TANF resource limit, so the correct answer is $0. The trap is the ""financial assistance"" line item in the household facts: it is an unearned-income input, not a TANF payment, and treating it as the answer skips the eligibility determination entirely. Answering this variable requires running the categorical test — dependent child present, resources under the limit — rather than pattern-matching a dollar figure labeled ""assistance"" in the inputs."
-us,scenario_091,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"The reference comes from AGI of $35,012.56 ($27,000 wages less only the $926 traditional 401(k), plus interest and dividends, less the $43 IRA deduction). From that it subtracts the $16,100 2026 OBBBA standard deduction (no personal exemption) and the $627.91 qualified overtime deduction. The $3,968 of qualified dividends falls in the 0% band, leaving $14,316.66 of ordinary income, taxed at 10% up to $12,400 and 12% above for $1,470. A $120 saver's credit (10% of all $1,199 in traditional and Roth 401(k)/IRA contributions) then brings it to $1,350. The AOTC is $0 because no qualified education expenses are listed. The wrong models fell into five traps: dropping the overtime deduction; using pre-OBBBA or TCJA-sunset parameters; treating the $6,589 ESI premiums as a pre-tax wage reduction; claiming an AOTC with zero expenses; and leaving out or understating the saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"The reference uses post-OBBBA 2026 law. AGI is $35,012.56 (wages net of the $926 traditional 401(k), plus interest and dividends, minus the $43 IRA deduction). Two deductions come off that: the $16,100 standard deduction and the $628 qualified-overtime deduction, which gives taxable income of $18,284.66. After the $3,968 of qualified dividends is pulled out at 0%, ordinary income of $14,316.66 is taxed at 10% up to $12,400 and 12% above it, for $1,470. A $120 nonrefundable Saver's Credit (10% of $1,199 in traditional plus Roth contributions) brings it to $1,350. The AOTC is $0 because no qualified tuition is listed, and ESI premiums are not excluded from wages. Most wrong answers made one of these mistakes: dropped the overtime deduction, dropped the Saver's Credit or cut its base, claimed an AOTC with no expenses, assumed the TCJA sunset, excluded the $6,589 of ESI premiums from wages, or used outdated standard-deduction and bracket amounts."
us,scenario_091,federal_refundable_credits,10,llm_error,other,False,,"The decisive step is the American Opportunity Credit's base: the credit is 100% of the first $2,000 plus 25% of the next $2,000 of qualified tuition and related expenses, and this household lists no tuition — an unlisted numeric input the prompt explicitly instructs be treated as $0 — so the credit is $0 and its 40% refundable portion is $0. The listed AOTC items (eligible institution, 1098-T/exception, institution EIN, at least half-time, pursuing a credential) are administrative gating conditions, not expenditures, and nine of ten models converted those flags into the statutory maximum $2,500 credit and booked $1,000 (or a mis-scaled $400/$1,200) of refundability. The EITC is separately $0 three times over: the head is 22, below the age-25 floor for a childless filer, and both the ~$36,000 AGI and the childless phase-out ceiling foreclose it; with no children there is no refundable CTC, no recovery rebate credit and no refundable payroll tax credit, leaving $0."
us,scenario_091,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Wisconsin Medicaid eligibility requires qualification through an applicable categorical pathway and its income rules. This 22-year-old nondependent falls into no Medicaid category, and MAGI of 2.19 times FPL does not satisfy an adult eligibility pathway, so immigration eligibility alone does not confer coverage."
us,scenario_091,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 6.2% Social Security plus 1.45% Medicare on the full $27,000 of gross wages, which comes to $1,674.00 + $391.50 = $2,065.50. The listed $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums do not lower the FICA wage base. The $628 FLSA overtime premium is already inside the $27,000 gross, and Wisconsin has no mandatory employee state payroll tax. Most of the wrong models treated the ESI premiums as a pre-tax Section 125 deduction and taxed only $20,411. The rest added the overtime twice, invented a state payroll tax, used the wrong combined rate, or made an arithmetic or reporting slip."
us,scenario_091,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_091,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference starts with wages net of the $926.16 401(k) deferral ($26,073.84), adds $4,806 interest and $4,176 dividends, leaves the $1,170 non-Schedule-D capital gain distributions out of gross income, and subtracts the $43.28 IRA deduction. That gives WI income of $35,012.56. Wisconsin's sliding-scale standard deduction at that income is $12,067.66, and the $700 personal exemption is also a deduction, not a credit. That leaves $22,244.90 of taxable income, taxed at 3.5% on the first ≈$15,010 and 4.4% above that, for $843.66. Most models put the $1,170 into income (often net of a 30% exclusion) and got the 2026 sliding standard deduction wrong: some treated it as fully phased out, some applied no phase-down, and others phased it down too steeply. Many also dropped the $700 exemption or used old bracket widths or pre-2023 rates. The rest zeroed out the tax with credits that don't apply here (AOTC, School Property Tax Credit) or subtracted ESI premiums, overtime, or health premiums from income. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)."
+us,scenario_091,state_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"The engine starts from Wisconsin AGI of $35,012.56. That is wages net of the $926.16 pre-tax 401(k) deferral, plus $4,806 of interest and $4,176 of dividends, minus the $43.28 IRA deduction. The $1,170 of non-Schedule-D capital gain distributions is left out entirely. Employer insurance premiums, other health premiums and overtime are not subtracted. It then subtracts the sliding-scale standard deduction of $12,067.66 and the $700 personal exemption, which gives $22,244.90 of taxable income, taxed at 3.5%/4.4% to get $843.66 with no nonrefundable credits. The models went wrong in four ways: (1) they included the $1,170 of gains, in full or at 70%; (2) they phased the standard deduction down too far, left it unphased, or dropped the $700 exemption; (3) they subtracted the $6,589 of employer insurance premiums or the $1,800 of health premiums from income; or (4) they invented nonrefundable credits (AOTC, school property tax credit) that cut the tax to $0. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)."
us,scenario_091,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_092,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Only the $14,980 taxable pension enters adjusted gross income; the Social Security benefit contributes no taxable amount under the federal provisional-income calculation. The age-77 single filer’s applicable standard deduction exceeds that AGI, leaving zero taxable income and therefore zero federal income tax before refundable credits."
us,scenario_092,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and disability do not independently establish Alabama Medicaid eligibility; the person must qualify through a specific Medicaid category. This person receives no SSI, has MAGI income of 2.03 times FPL, and qualifies through none of the available pathways, so the Medicaid category is NONE and eligibility is No."
@@ -716,10 +715,10 @@ us,scenario_092,state_income_tax_before_refundable_credits,23,llm_error,taxable_
us,scenario_092,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_093,dependent1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not confer Missouri Medicaid eligibility. The person must satisfy a specific non-MAGI pathway's categorical and financial requirements; this 27-year-old dependent receives no SSI, qualifies for no non-MAGI category, and at 2.82 times FPL exceeds the applicable MAGI limits."
us,scenario_093,dependent1_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's is_medicare_eligible turns on a single age test — age 65 or older — with the narrow ESRD/ALS entitlements not modeled and no disability branch at all; Dependent 1 is 27, so the variable is False. Every wrong model substituted the household's bare ""is disabled"" flag for that age test, asserting a disability-based Medicare pathway either as a PolicyEngine rule or as a real-world shortcut. The statutory disability pathway requires 24 months of SSDI entitlement, and Dependent 1 has $45,000 in annual wages (~$3,750/month), far above substantial gainful activity, plus no listed SSDI receipt — the prompt directs unlisted benefit receipt to be treated as zero, so no disability pathway is reachable on these facts."
-us,scenario_093,federal_income_tax_before_refundable_credits,45,llm_error,household_unit_or_filing_status,False,,"The single tax unit is a married-filing-jointly return, with the permanently disabled 27-year-old claimed as a dependent. Only the head's and spouse's income counts: wages net of the $4,058.12 traditional 401(k), plus $1,156 interest, minus $2,500 student loan interest and $189.62 IRA. That gives AGI of $112,408.25 and taxable income of $80,208.25 after the $32,200 2026 standard deduction, $9,128.99 of tax under the permanent TCJA/OBBBA brackets (10% to $24,800, then 12%), and $8,628.99 after the $500 credit for other dependents. Most wrong answers either added the dependent's $45,000 wages to the joint return or added a separate single-return tax for the dependent. Others subtracted employer-sponsored insurance (ESI) premiums from wages, denied the credit for other dependents (ODC) by applying the qualifying-relative gross income test to a disabled qualifying child, assumed the TCJA had expired, or took an auto-loan-interest deduction."
+us,scenario_093,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,household_unit_or_filing_status,False,,"The joint return counts only the head's and spouse's income. Wages of $118,000, minus $4,058 of traditional 401(k) deferrals, plus $1,156 of interest, minus a $190 traditional IRA deduction and $2,500 of capped student loan interest, give AGI of $112,408. The 2026 standard deduction of $32,200 leaves $80,208 of taxable income, taxed at 10%/12% for $9,129, and the $500 credit for other dependents for the disabled 27-year-old brings it to $8,629. The disabled adult is a qualifying child of any age, so no gross-income or support test removes the credit. Most models went wrong in one of these ways: they put the dependent's $45,000 wages on the joint return, added a separate tax on the dependent, denied the $500 credit, used TCJA-sunset or 2025 parameters, subtracted ESI premiums from wages, or took an auto-loan interest deduction that doesn't apply. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_093,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_093,payroll_tax,18,llm_error,payroll_tax_base,False,,"The reference is 7.65% of the full $163,000 in gross wages ($53,000 + $65,000 + $45,000). That is 6.2% Social Security ($10,106), since every earner is below the 2026 wage base, plus 1.45% Medicare ($2,363.50). There is no Additional Medicare Tax and no Missouri employee payroll tax. The main trap is the Section 125 assumption: eight models subtracted each worker's $8,389 of employer-sponsored insurance premiums from FICA wages and got $10,544.22 (7.65% of $137,833). No listed fact makes those premiums salary-reduction cafeteria-plan contributions, so the listed gross wages are the FICA base. Most of the remaining models used the right base and rates, and several showed $12,469.50 in their own working, but they added up wrong or submitted a different number from the one they computed."
-us,scenario_093,state_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"Only the spouses' income enters the Missouri tax base. The disabled 27-year-old is claimed on the couple's return inside the single tax unit, so the dependent's $45,000 of wages does not join the couple's AGI and does not create a separate Missouri return. From MO AGI of $112,408.25, Missouri allows the $32,200 standard deduction and a federal-tax deduction of 5% ($431.45), because the AGI falls in the $100,001–$125,000 band. That AGI already subtracts the 401(k) and IRA contributions and applies the $2,500 student-loan interest cap; ESI premiums are not subtracted. Missouri then splits the $79,776.80 of taxable income between the spouses by income percentage and applies the 2026 schedule (4.7% top rate) to each share, giving $1,452.33 + $1,935.91. Most models went wrong in one of these ways: taxing the dependent, running one schedule on the joint total, skipping the 5% federal-tax deduction, or using stale rates or deductions or exemptions that don't exist."
+us,scenario_093,payroll_tax,19,llm_error,payroll_tax_base,False,,"Employee FICA here is 7.65% of each worker's full listed gross wages: 6.2% Social Security plus 1.45% Medicare on $53,000, $65,000 and $45,000, all below the wage base. That gives $3,286 + $768.50, $4,030 + $942.50 and $2,790 + $652.50, for a total of $12,469.50. No Additional Medicare Tax applies, and Missouri has no employee state payroll tax. The facts never say the $8,389 ESI premiums were paid through a pre-tax cafeteria plan. The largest group of models (8) assumed a Section 125 salary reduction anyway, cutting FICA wages to $137,833 and the tax to $10,544.22. Most other models wrote out the correct $12,469.50 components or formula but then added wrong, multiplied wrong, or submitted a different number from the one they computed."
+us,scenario_093,state_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The engine puts the 27-year-old disabled dependent in the couple's tax unit with $0 Missouri AGI. Only the spouses' $112,408.25 is taxed. That figure reflects excluding 401(k) deferrals, the $190 IRA deduction and the $2,500 student-loan-interest cap. Missouri taxable income is $79,776.80 after the $32,200 2026 MFJ standard deduction and a federal-tax deduction of 5% of federal tax ($431.45), which applies at MO AGI of $100,001–$125,000. That income is split between the spouses by income share, and the 4.7%-top-rate schedule is applied to each share separately, so each spouse gets the lower-bracket savings. Most wrong answers taxed the dependent's $45,000 of wages, either in joint AGI or on a separate single return, which pushed AGI past $125,000 and eliminated the federal-tax deduction. Other wrong answers used made-up personal exemptions, outdated rates or deductions, subtracted employer insurance premiums from wages, or applied the schedule only once to the joint total. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_095,federal_income_tax_before_refundable_credits,3,llm_error,household_unit_or_filing_status,False,,"The head's surviving-spouse status determines the federal filing-status pathway and supplies a standard deduction large enough to eliminate the $15,286.89 of AGI. Social Security is not taxable at this provisional-income level, so it is excluded from AGI; after the half-self-employment-tax deduction, taxable income and income tax before credits are both zero."
us,scenario_095,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Federal refundable credits include only the refundable credit components applicable under 2026 law. Age 73 does not independently create a $1,200 refundable senior credit, and all five applicable components—EITC, refundable American Opportunity Credit, refundable CTC, Recovery Rebate Credit, and refundable payroll tax credit—equal zero."
us,scenario_095,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -729,7 +728,7 @@ us,scenario_095,head_medicare_eligible,1,parse_contract_failure,missing_output,F
us,scenario_095,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_095,payroll_tax,2,llm_error,payroll_tax_base,False,,"The payroll_tax output includes employee-side taxes on wages and mandatory employee state payroll contributions, but the household has no wage or salary income. Self-employment income belongs in the separate self_employment_tax output and does not enter the employee payroll-tax base, so payroll_tax is $0."
us,scenario_095,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_095,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax here is a single mechanical chain: net earnings from self-employment are 92.35% of the $4,198 of net profit ($3,876.85), that base clears the $400 §1402(b) floor, and the combined 15.3% rate (12.4% OASDI + 2.9% HI) on it produces the reference $593.23. Nothing in the household facts alters it — age 73, Social Security receipt, the 401(k) distribution, and the surviving-spouse status are all irrelevant to §1401 liability, and the above-the-line deduction for half of SE tax reduces income tax, never the SE tax itself. The six wrong answers each break one link: two zero out the tax (one by inverting the $400 comparison, one by treating the half-SE-tax deduction as an offset against SE tax), two mangle the 92.35% base factor (rounding it to 92%, or omitting it and taxing gross $4,198), one submits a number contradicting its own correct arithmetic, and one returns nothing."
+us,scenario_095,self_employment_tax,7,llm_error,payroll_tax_base,False,,"The correct method has two steps. First, reduce self-employment income to net earnings from self-employment by multiplying by 0.9235, which gives about $3,877. That amount is above the $400 filing floor and well below the Social Security wage base. Second, apply the full 15.3% rate, which gives about $593. Some models left out the 0.9235 adjustment. Others compared the wrong figure to the $400 floor or treated the income-tax deduction for half of SE tax as a reduction of SE tax itself. Others did the arithmetic and then submitted a different number. The head's age of 73 and receipt of Social Security do not exempt them from SE tax."
us,scenario_095,snap,5,llm_error,thresholds_rates,False,,"The household’s $29,605 annual income exceeds the applicable 2026 New Jersey SNAP gross-income eligibility limit for one person, so the SNAP calculation stops with no eligibility and a $0 allotment. Broad-based categorical eligibility, elderly-household deductions, and the minimum-allotment rule do not produce a benefit after this income screen fails."
us,scenario_095,state_income_tax_before_refundable_credits,22,llm_error,thresholds_rates,False,,"New Jersey imposes no income tax when NJ gross income, which excludes Social Security and is measured after the age-62+ pension exclusion, is $10,000 or less for single filers, or $20,000 or less for qualifying widow(er)s and the other joint-type statuses. For this 73-year-old, NJ gross income is $15,583 minus the $11,385 401(k) exclusion, or $4,198. That is below either threshold, so the tax is $0. Almost every model correctly excluded Social Security and the 401(k) distribution. They then treated the leftover $4,198 of self-employment income as taxable, took off exemptions and sometimes medical expenses, and applied the 1.4% bracket. None applied the gross-income threshold that zeroes the liability."
us,scenario_095,state_refundable_credits,4,llm_error,credit_phaseout,False,,"This household has no children, so the NJ EITC is its only possible refundable credit. For a childless filer aged 65 or older, New Jersey computes the credit as 40% of a hypothetical federal childless EITC. The phase-in gives 7.65% × earned income of about $3,901 ($4,198 SE income minus $296.58, the deductible half of SE tax), or $298.45. The phase-out then takes 7.65% × (AGI − non-joint threshold). AGI is about $15,286 because it includes the $11,385 401(k) distribution (Social Security is untaxed because provisional income is below $25,000). Since the 401(k) distribution alone exceeds the roughly $11k threshold, the reduction is larger than the phase-in amount and the credit is $0. The household paid no property tax or rent, so the NJ property tax credit is also $0, and ANCHOR is not an income tax credit."
@@ -758,15 +757,15 @@ us,scenario_099,child2_head_start_eligible,2,llm_error,categorical_eligibility,F
us,scenario_099,child2_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility requires both categorical eligibility and financial eligibility; being age four satisfies only the child-category requirement. The household's approximately $159,676 of annual income exceeds California's WIC limit of 185% of the federal poverty level for a four-person household, so child 2 is not eligible."
-us,scenario_099,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference uses 2026 law as set by OBBBA. That means a $32,200 standard deduction plus the new $2,000 joint deduction for charitable gifts by non-itemizers, which gives taxable income of $125,385.05. The 2026 brackets are 10% to $24,800, 12% to $100,800, then 22%, which gives $16,999.75 of tax. The full $2,200-per-child CTC ($4,400) is used against that liability. The CDCC is 32% of $6,000 ($1,920): OBBBA's rate falls from 35% by one point per $4,000 of joint AGI over $150,000. The models most often took the CDCC at the old 20% rate, left out the $2,000 charitable deduction, used a $2,000 CTC, or treated only a sliver of the CTC as nonrefundable. A large group applied expired pre-TCJA law (personal exemptions, 15/25% brackets, a $1,000 CTC phased out above $110,000), even though OBBBA made the TCJA structure permanent. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 10,679.75; the corrected value is 10,711.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_099,federal_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"For 2026, the reference uses permanent OBBBA law. Under it, the MFJ standard deduction is $32,200 and non-itemizers also get a $2,000 charitable deduction. The CTC is $2,200 per child and is fully nonrefundable up to tax liability. The CDCC uses the enhanced rate schedule, which steps down from 35% by 1 point per $4,000 of joint AGI over $150,000, giving 32% × $6,000 = $1,920. The traditional IRA is not deductible because both spouses are covered by workplace plans, and the auto-loan interest deduction does not apply. Most wrong answers used one of these: TCJA-sunset rules (personal exemptions, $1,000 CTC with phaseout, 10/15/25 brackets); the old 20% CDCC rate; no $2,000 non-itemizer charitable deduction; a $2,000 CTC; or only $300–$400 per child of CTC treated as nonrefundable."
us,scenario_099,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The trap is §24(d)'s liability-limitation prong: the refundable CTC is the *lesser* of the earned-income formula (15% of earnings over $2,500), the per-child refundable cap, and the portion of the CTC left unused after the credit offsets tax liability. This married-filing-jointly household has AGI near $159,676 (wages $165,000 plus $1,010 interest, $144 dividends, capital netting limited to a $3,000 loss, less $3,232 of traditional 401(k)/IRA contributions and the $338 educator expense) and roughly $127,000 of taxable income after the 2026 MFJ standard deduction, producing about $17,400 of federal income tax before credits — far more than the $4,400 of 2026 OBBBA CTC ($2,200 × 2), so the entire CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is zero because $159,676 is far beyond the two-child MFJ phase-out, and the CDCC is nonrefundable. Every wrong model treated a refundability cap or the earned-income formula as a standalone entitlement instead of applying the unused-credit ceiling."
-us,scenario_099,free_school_meals_eligible,38,llm_error,state_local_rule,False,,"California runs a universal school meals program: since 2022-23 every public K-12 student gets free breakfast and lunch whatever the household income. PolicyEngine therefore sets school_meal_tier to FREE for Child 1, the 9-year-old K-12 student, even though the household's FPG ratio is 5.03 and it has no SNAP or TANF categorical eligibility. Every wrong model applied the federal NSLP income test, which requires income at or below 130% of FPL for free meals. None applied the state universal-meals override, and all concluded No."
+us,scenario_099,free_school_meals_eligible,39,llm_error,state_local_rule,False,,"California runs universal free school meals (the CA Universal Meals Program, Education Code §49501.5, in effect since the 2022-23 school year). Every enrolled K-12 student gets free breakfast and lunch regardless of income. Because of this, PolicyEngine sets school_meal_tier to FREE even though the household's income is 5.03 times the poverty guideline and it has no categorical eligibility. The 9-year-old is in K-12, which produces $1,130.96 a year in free-meal support. Every model that answered applied only the federal NSLP 130%-of-FPL income test, so each one returned No by missing the state's universal program. Kimi-k3 gave no answer."
us,scenario_099,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_099,payroll_tax,40,llm_error,state_local_rule,False,,"Federal FICA is straightforward here: both earners are under the Social Security wage base and the pair is under the $250,000 MFJ Additional Medicare threshold, so SS is $10,230 and Medicare is $2,392.50 on the full $165,000 of gross wages. What separates the reference from the wrong answers is California's mandatory employee SDI. For 2026 it is 1.3% of all wages with no cap, which is $2,145. Models either left SDI out (12,622.50), used an older SDI rate (1.1% gives $14,437.50, 1.2% gives $14,602.50), or wrongly took the employer-sponsored insurance premiums out of the FICA/SDI wage base."
+us,scenario_099,payroll_tax,41,llm_error,state_local_rule,False,,"Federal FICA was easy: 7.65% of the $165,000 gross wages gives $12,622.50, and there is no Additional Medicare Tax. Most models got that part right. The missing piece is California's mandatory employee SDI contribution. In 2026 it is 1.3% of all wages with no wage cap, which adds $2,145 and brings the total to $14,767.50. Wrong answers fall into four groups: models that left out SDI entirely, models that used the outdated 1.1% (2024) or 1.2% (2025) rate, models that subtracted the employer-sponsored insurance premiums from the wage base even though the reference taxes gross wages, and models whose final number did not match their own working."
us,scenario_099,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -775,42 +774,42 @@ us,scenario_099,spouse_medicaid_eligible,1,parse_contract_failure,missing_output
us,scenario_099,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_099,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"CA AGI is $159,585. That is gross wages less the $3,088 in 401(k) deferrals, the $144 IRA deduction and the $338 educator deduction, plus interest and dividends, with the capital loss limited to $3,000. The household itemizes $20,477: $11,823 of charity, which is $12,621 minus the 0.5%-of-AGI charitable floor, plus $8,654 of property tax. There is no medical deduction because medical costs of $10,000 ($11,400 with over-the-counter items) are below the 7.5%-of-AGI floor of about $11,969. That leaves $139,108 of taxable income, which reaches only the 8% bracket. The frozen reference prices it on the engine's projected 2026-indexed MFJ schedule, $5,778 of tax, and subtracts $1,284.46 of projected personal and dependent exemption credits for $4,493.74; the release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $5,896.78 less $1,256 on the same income. The wrong answers came from a few repeated mistakes: counting the $2,500 premium twice to claim a medical deduction, subtracting the $24,597 of employer-sponsored insurance premiums from income, taking the standard deduction instead of itemizing, using 2024 brackets rather than the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, ignoring the 0.5% charitable floor, and getting the exemption credits wrong or leaving them out. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)."
+us,scenario_099,state_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's California itemized deduction reuses the federal charitable_deduction, and for 2026 that deduction already has the OBBBA 0.5%-of-AGI floor (IRC §170(b)(1)(I)) applied. California does not adopt that floor because its IRC conformity date under R&TC §17024.5 is earlier, so the CA charitable deduction is cut by $797.92, worth about $64 of tax.","The engine itemizes $20,476.57: real estate taxes of $8,653.50 plus charitable gifts of $11,823.08 after a 0.5%-of-AGI floor. It allows no medical deduction because out-of-pocket medical costs (at most $11,400) are below the 7.5% floor of about $11,969. That leaves $139,108.47 of taxable income, which the 2026 indexed MFJ brackets tax at $5,778.21; subtracting $1,284.46 of exemption credits (two personal, two dependent) gives $4,493.74. The wrong answers did one or more of these: took the standard deduction, created a medical deduction by counting the two premium fields twice, removed employer-sponsored insurance premiums from AGI, got the exemption credits wrong, or used unindexed or inflated brackets. The closest answers deducted charity without the 0.5% floor. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)."
us,scenario_099,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_100,child1_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child not to qualify for Medicaid. At age 6 and this income, Child 1 qualifies for Montana Medicaid under the OLDER_CHILD category, so Medicaid eligibility categorically precludes CHIP eligibility; the CHIP income ceiling and existing employer-sponsored insurance do not alter that result."
-us,scenario_100,child1_head_start_eligible,5,llm_error,age_disability,False,,Head Start's preschool-age test excludes a child whose benchmark age is 6; low household income does not override that age requirement. The models treated the income test as sufficient after improperly extending eligibility beyond the program's age boundary.
+us,scenario_100,child1_head_start_eligible,6,llm_error,age_disability,False,,"Head Start for preschool-age children in PolicyEngine depends on an age window of 3 up to school entry. A 6-year-old is school-age, so Child 1 fails the age test before income matters. The household's very low income ($5,915 in wages, well under the poverty line) would meet the income test, but income cannot make up for failing the age test. Five models let low income override the age limit and stretched the preschool window to cover a 6-year-old. The sixth model got the age rule right but submitted a value that contradicted its own explanation."
us,scenario_100,child1_medicaid_eligible,3,llm_error,health_coverage,False,,"Montana determines this 6-year-old's Medicaid eligibility through the older-child MAGI pathway, whose income limit is 143% FPL. The child's household MAGI is 0.07 × FPL, so the child meets the income test, and employer-sponsored insurance does not negate eligibility."
us,scenario_100,child1_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is restricted to children under age 5; Child 1 is age 6 and therefore fails the categorical age requirement. The traced WIC income measure of approximately $5,915 also exceeds the applicable 185%-of-FPL limit, so the child fails the income test as well."
us,scenario_100,child2_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, not merely under the CHIP income ceiling and uninsured. At age 5 and this household income, child2 qualifies for Medicaid under the YOUNG_CHILD category, so Medicaid eligibility precludes CHIP eligibility."
us,scenario_100,child2_early_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Early Head Start serves infants and toddlers under age 3, while Head Start is the separate program for preschool-age children. Child 2 is age 5, so low household income does not overcome the Early Head Start age restriction."
us,scenario_100,child2_head_start_eligible,4,llm_error,age_disability,False,,"Head Start for preschool-age children covers children from age 3 through compulsory school age, so a 5-year-old is squarely inside the band — it is the 6-year-old Child 1 who falls outside it, and Early Head Start that stops below age 3. On top of that, the household's TANF receipt of $505.41 monthly confers categorical eligibility on all members, and its $1,875.40 adjusted gross income is far below the poverty guideline for three people, so income eligibility holds independently. Three models cut Head Start off at age 4 and one defaulted the flag to zero with no derivation; all four therefore never applied the categorical TANF pathway or the income test that both resolve to eligible."
us,scenario_100,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Montana Medicaid applies a young-child eligibility pathway to the 5-year-old dependent. The child's household MAGI is 0.07 times FPL, below the applicable young-child income threshold, so the eligibility test returns Yes."
-us,scenario_100,child2_wic_eligible,11,llm_error,categorical_eligibility,False,,"WIC child categorical eligibility applies only to children under age five; a child whose stated age is 5 has reached the fifth birthday and is outside the child category. The prompt holds age constant for the full year and supplies no existing certification-period status, so low household income cannot establish eligibility after the categorical test fails."
-us,scenario_100,federal_refundable_credits,35,reference_engine_defect,credit_phaseout,False,,"The reference uses gross wages of $5,915 as earned income for both credits. Those wages already include the $710 FLSA overtime premium. The traditional 401(k) deferral, the IRA contributions and the health premiums do not reduce this figure. With two children, the 40% EITC phase-in gives $2,365.89. The refundable CTC is 15% × ($5,915 − $2,500) = $512.21, using the 2026 $2,500 threshold that OBBBA made permanent, for a total of $2,878.10. The largest group of wrong answers subtracted the $3,859 401(k) deferral, leaving $2,056 of earned income, which gives $822.40 of EITC and no refundable CTC. The second group used the pre-TCJA $3,000 CTC threshold and got $437.25 instead of $512. The rest either returned zero, double-counted overtime, dropped the earned-income limit on the refundable CTC, or submitted a number different from the one they had derived. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
+us,scenario_100,child2_wic_eligible,12,llm_error,age_disability,False,,"WIC's child category covers children from age 1 until their 5th birthday, so the age test is strictly under 5. Child 2 is age 5, has passed that birthday, and is not in the child category. They are also not an infant, pregnant, postpartum, or breastfeeding, so they fall into no WIC category. The household's very low income doesn't matter once the categorical age test fails. Every model that answered treated age 5 as within the WIC child range, and none of them applied the strict under-5 cutoff."
+us,scenario_100,federal_refundable_credits,36,reference_engine_defect,credit_phaseout,False,"PolicyEngine computes EITC and ACTC earned income from gross employment_income ($5,915) without subtracting the $3,859 traditional 401(k) elective deferral. IRC §32(c)(2)(A)(i), as amended by EGTRRA 2001 and incorporated into §24(d), counts employee compensation only ""if such amounts are includible in gross income."" IRS Pub 596 likewise says employee pay counts as earned income only if it is taxable. That gives $2,056 of earned income, an EITC of 40% × $2,056 = $822.40 and an ACTC of $0, not $2,878.10.","The reference applies the 40% two-child EITC phase-in rate to the full $5,915 gross wages, which gives $2,365.89. It adds an ACTC of 15% × ($5,915 − $2,500), which gives $512.21. The main split is over the earned-income base: eight models subtracted the $3,859 traditional 401(k) deferral and got $822.40 with no ACTC. The other wrong answers came from model-specific errors: using the expired $3,000 ACTC threshold, multiplying the earnings-based ACTC limit per child, counting the $710 overtime premium twice, treating ESI premiums as pre-tax wage reductions, ignoring the earnings limit on the refundable CTC, or submitting numbers that contradict the model's own work. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
us,scenario_100,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Free school meals are awarded through two independent pathways, and this Montana household clears both: it receives $8,625.89 of SNAP, which categorically confers free-meal status, and its countable income of $11,979.68 ($5,914.72 wages plus $6,064.96 TANF) is 44% of the federal poverty guideline for a three-person household, far under the 130% free-meal ceiling. The two children, ages 5 and 6, are both school-age, so the free tier pays a $7.15 daily per-child subsidy for the contiguous-US school year, $2,261.92 annually. The trap is that SNAP and TANF receipt are derived outputs of this same household, not listed input facts, so a model that only credits explicitly stated benefit receipt loses the categorical pathway — but even the wage-only figure of $5,915 sits at roughly 22% of the poverty guideline, so the income test alone still lands in the free tier. Both models returned 0 without running either test."
-us,scenario_100,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The head qualifies through Montana’s parent/caretaker Medicaid category because the household includes dependent children and MAGI equals 0.07 × FPL, below the category’s 100% FPL limit. This MAGI pathway does not apply an asset test, so the bank balance and employer-sponsored insurance do not negate eligibility."
+us,scenario_100,head_medicaid_eligible,4,llm_error,thresholds_rates,False,,"The head is a 46-year-old parent of two young children with MAGI of about $5,915, which is 0.07 × FPL for a household of three. That is far below Montana's Medicaid limit for the parent/caretaker category, so the head is eligible. MAGI-based Medicaid has no asset test, and having employer-sponsored insurance does not bar eligibility. The wrong models used an invented FPL dollar figure, applied an asset test, or treated ESI as disqualifying. None of them compared 7% of FPL against the parent income limit."
us,scenario_100,head_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility below age 65 does not follow from a current disability flag alone; it requires a qualifying Medicare pathway, such as entitlement after the applicable Social Security disability waiting period or eligibility based on end-stage renal disease. The 46-year-old head has no listed Medicare-qualifying entitlement or condition, so the disability input does not change the result from ineligible."
us,scenario_100,head_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility is individual, not household-wide: an adult qualifies only as a pregnant, breastfeeding, or postpartum woman, while merely caring for a young child does not confer eligibility. The 46-year-old head has no qualifying WIC status, and neither low household income nor a child's presence satisfies the head's categorical requirement."
us,scenario_100,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_100,payroll_tax,7,llm_error,payroll_tax_base,False,,"The trap is the payroll-tax wage base: PolicyEngine assesses employee OASDI at 6.2% and HI at 1.45% on the full $5,915 of employment income, giving $366.71 + $85.76 = $452.48. None of the listed health inputs touch that base — the $20,644 employer-sponsored insurance premium and the two $564 individually paid premiums are health-coverage and medical-expense inputs, not Section 125 salary reductions — and the $710 FLSA overtime premium is already inside the annual gross wage figure rather than additional to it. Traditional 401(k) and IRA amounts also leave FICA wages untouched, and Montana imposes no mandatory employee-side state payroll tax, so the answer is exactly 7.65% of wages. The wrong answers split into base errors (subtracting premiums to $4,787 or to $0, or adding the overtime premium to reach $6,625) and, for the three models that found the right base, an arithmetic slip, an invented rounding, and a value/explanation mismatch."
us,scenario_100,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier. This household is at 44% of the federal poverty guideline and also meets categorical eligibility, so its school_meal_tier is FREE and reduced_price_school_meals_eligible is 0."
-us,scenario_100,snap,46,prompt_ambiguity,age_disability,False,,"The reference comes from three income and deduction steps that partly cancel out. First, SNAP gross income is $998.31/month: $492.92 of wages plus $505.39 of Montana TANF cash, which is the $725.08 payment standard minus 75% of earnings above $200. Second, the head's medical costs are not deductible because the head gets no SSI or SSDI: $2,800 in the bank is over SSI's $2,000 resource limit. So the head does not meet SNAP's disability definition in 7 U.S.C. 2012(j). Third, although no rent is listed, the engine grants the heat-and-eat standard utility allowance to a household without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended (engine defect r30_snap_heat_and_eat_sua, not fixed upstream), so the frozen reference takes a $453.64 excess shelter deduction from the utility allowance. That leaves $237.09 net income, a $71.10 contribution and $713.90/month for January–September (at the $785 maximum); the frozen reference's $733.60/month for October–December uses the engine's projected FY2027 schedule, whereas the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule, including the $785 maximum, for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $713/month ($8,556). Most models counted wages only, took a disabled-member medical deduction and took no shelter deduction, landing at $9,000–$9,420. The record's correction of r30_snap_heat_and_eat_sua also takes no utility-allowance shelter deduction and, with TANF counted and no medical deduction, gives $691 net income, a $208 contribution and $577/month, moving the output from $8,556 to $6,924. Models that did count TANF also took no shelter deduction, as that correction does, and came in above its $6,924; most came in below the frozen reference. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification."
+us,scenario_100,snap,47,prompt_ambiguity,age_disability,False,,"SNAP gross income is $998.31 a month: $492.92 in wages plus about $505.39 in Montana TANF cash, which counts as unearned income. The deductions are the $209 standard deduction, the $98.58 earned income deduction (20%), and a $453.64 excess shelter deduction. The shelter deduction comes from Montana's standard utility allowance, which counts as a shelter cost even though no rent is listed. There is no excess medical deduction because the head receives no disability benefit such as SSI or Social Security disability, so the head does not count as disabled for SNAP. Net income is $237.09, and 30% of that ($71.10) is subtracted from the maximum allotment. That gives $713.90 a month for January–September and $733.60 for October–December, when the FY2027 allotment of $802.60 applies. Most models used wages only, left out TANF, added a medical deduction, and landed near the maximum allotment ($9,000–$9,420). The models that counted TANF left out the shelter deduction and landed around $7,000–$7,800. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification."
us,scenario_100,ssi,9,llm_error,categorical_eligibility,False,,"The generic `is disabled` household fact does not make the head categorically eligible for SSI under PolicyEngine; the controlling SSI-specific indicator is `is_ssi_aged_blind_disabled=False`. All three individuals fail the aged-65-or-older, blind, or qualifying-disabled requirement, so income exclusions, resources, and benefit-rate calculations are never reached and household SSI is $0."
-us,scenario_100,state_refundable_credits,44,reference_engine_defect,credit_phaseout,False,,"Montana's only refundable credit for this household is the state EITC, and for 2026 it equals 20% of the federal EITC. The federal EITC is the two-child 40% phase-in applied to gross earned wages of $5,915, which gives $2,365.89; traditional 401(k) deferrals do not reduce that base. 20% × $2,365.89 = $473.18. Most wrong answers fall into three groups. One group says Montana has no refundable EITC, or no income tax at all, and answers $0. A second group uses the old 3% or 10% rate, which gives $70.98 or $236.60. A third group shrinks the federal EITC base by subtracting the 401(k) deferral, which gives $822.40. Several models also added a $1,200 young-child credit that is not part of this household's 2026 Montana refundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
-us,scenario_100,tanf,46,llm_error,thresholds_rates,False,,"Montana TANF for this three-person unit (parent plus children aged 6 and 5) uses a monthly payment standard of $725.10 ($8,701.21 a year). From that it subtracts countable earnings: ($5,915/12 − $200) × 75% = $219.69. That leaves $505.41 a month, or $6,064.96 a year. Resources pass: the $2,800 bank balance is under the $3,000 limit, and the vehicle is excluded. SSI is $0 because the $2,800 exceeds SSI's $2,000 resource limit, so no SSI income offsets TANF. Most nonzero answers used an outdated $588 (or other low) payment standard, or a 50% earnings disregard instead of 25%. The $0 answers invented disqualifiers: asset limits below $3,000, counting the vehicle, imputed SSI, double-counted overtime, or a claim that Montana TANF is not modeled."
-us,scenario_101,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference is simple ordinary-income tax on $85,030 of taxable income: AGI of $101,130 ($103,000 wages minus the $1,870 capital loss) less the 2026 single standard deduction of $16,100. The rates are 10% to $12,400, 12% to $50,400 and 22% above, which gives $13,418.60 with no nonrefundable credits. The wrong answers fall into three groups. Eight models assumed the TCJA would sunset and used the $8,300 standard deduction, the $5,300 personal exemption and 10/15/25 brackets, but OBBBA made the TCJA structure permanent. Seven models subtracted the listed $1,400 of auto loan interest as an OBBBA qualified vehicle-loan-interest deduction, although the facts establish no qualifying new, U.S.-assembled vehicle. The rest used stale or invented standard-deduction and bracket figures, or submitted numbers that contradict their own arithmetic."
+us,scenario_100,state_refundable_credits,45,reference_engine_defect,credit_phaseout,False,,"Montana's state refundable credits for this household come only from the Montana EITC. For 2026 that credit is refundable and equals 20% of the federal EITC. The federal EITC is the 40% two-child phase-in rate applied to the head's full $5,915 of earned income, which gives $2,365.89, so the Montana credit is $473.18. Most models went wrong in one of three ways: they said Montana has no state EITC, no income tax, or only a nonrefundable EITC (giving $0); they used the older 3% or 10% rate; or they cut earned income by the $3,859 traditional 401(k) deferral, which gives a federal EITC of $822.40. A few also added a $1,200 young-child credit that PolicyEngine does not count in this household's Montana refundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
+us,scenario_100,tanf,47,llm_error,categorical_eligibility,False,,"The family of three qualifies for Montana TANF. Its $2,800 in bank assets is under the $3,000 resource limit, and the one vehicle is excluded. The head gets no SSI because $2,800 is over the $2,000 SSI resource limit, so there is no unearned income to count. Countable earnings are 0.75 × ($5,915/12 − $200) = $219.69 a month, after the $200 disregard plus 25% of the rest. That amount is subtracted from Montana's 2026 three-person payment standard of about $725.10 a month, which gives about $505.41 a month, or $6,064.96 a year. Most wrong answers either returned $0 by inventing reasons the family was ineligible (no modeling, asset limits that are too low, SSI receipt, or counting overtime twice) or used an outdated payment standard around $588 or a 50% earnings disregard."
+us,scenario_101,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"The correct path is: AGI of $101,130 ($103,000 wages minus the $1,870 capital loss; the pension and veterans benefits are not taxable), minus the $16,100 2026 single standard deduction and nothing else, gives taxable income of $85,030. Under the 2026 brackets that OBBBA made permanent (10% to $12,400, 12% to $50,400, then 22%), that is $1,240 + $4,560 + $7,618.60 = $13,418.60, with no nonrefundable credits. The wrong answers fall into three groups. One group applied the TCJA-sunset rules ($8,300 standard deduction plus a $5,300 personal exemption, with 15%/25% brackets). A second group used 2025 or guessed standard deductions and bracket thresholds. A third group subtracted the $1,400 auto loan interest even though nothing in the facts shows a qualified passenger vehicle loan. Several models reached $13,418.60 or $85,030 and then reported a different number."
us,scenario_101,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_101,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security tax plus 1.45% Medicare tax on the full $103,000 of wages, with no Additional Medicare Tax or mandatory Texas employee payroll tax. This produces $6,386 plus $1,493.50, totaling $7,879.50; health-insurance premiums do not reduce the payroll-tax base unless established as pre-tax employer-plan deductions."
+us,scenario_101,payroll_tax,8,llm_error,other,False,,"The correct calculation is simple. Employee Social Security is 6.2% of the full $103,000 of wages, or $6,386, because the wages are below the 2026 wage base. Employee Medicare is 1.45%, or $1,493.50. No Additional Medicare Tax applies below $200,000, and Texas has no employee state payroll tax, so the total is $7,879.50. Most models reached exactly $7,879.50 in their reasoning and then submitted a different number with a made-up ""adjustment."" One model treated the listed health insurance premiums as a pre-tax payroll exclusion, another submitted zero, and one gave no output."
us,scenario_101,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident has $0 of state income tax before refundable credits regardless of the listed taxable-income deductions or credits. Taxes from other jurisdictions do not apply because the household resides in Texas."
-us,scenario_102,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The correct path uses the OBBBA-permanent TCJA structure for 2026. The married-filing-jointly standard deduction is $32,200 and the 10% bracket ends at $24,800. Gross wages of $87,200 plus $1 of self-employment income are fully taxable; the $7,989 ESI premium is not a stated salary reduction. The only other deduction is a $0.20 QBI deduction, which leaves $55,000.80 of taxable income, tax of $2,480 + 12% × $30,200.80 = $6,104.10, and no nonrefundable credits. Most models went wrong in one of four ways: they carried pre-OBBBA 2025 amounts ($30,000–$30,800 standard deduction, a $23,850–$24,500 10% bracket) forward; they assumed the TCJA sunset (personal exemptions and a 15% bracket); they subtracted the ESI premium from AGI; or they submitted a number that contradicts their own arithmetic."
+us,scenario_102,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 MFJ standard deduction and a 10% bracket that ends at $24,800. Gross income is $87,201 and the QBI deduction is $0.20, so taxable income is $55,000.80. The tax is then $2,480 + 12% × $30,200.80 = $6,104.10, and no nonrefundable credits apply. The listed wages already are the gross amounts, so the $7,989 employer-sponsored insurance premium is not subtracted from them. Most models either used pre-OBBBA inflation projections ($30,000–$30,840 standard deduction with a $23,850–$24,500 bracket), assumed the TCJA had expired and used personal exemptions with a 15% bracket, wrongly excluded the ESI premium, or reported a final number that did not match their own math."
us,scenario_102,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"Refundable federal credits require a qualifying statutory pathway; disability alone does not create a refundable federal income tax credit. With no qualifying children, education expenses, or other credit-triggering facts, and income above the childless EITC range, every refundable component is zero."
us,scenario_102,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish North Carolina Medicaid eligibility; the person must qualify through a specific MAGI or traditional categorical pathway and satisfy that pathway's financial rules. The head's household MAGI is $87,201, or 4.03 times FPL, and the engine assigns medicaid_category = NONE, so medical expenses, rent, and $1,001 in assets do not create eligibility."
us,scenario_102,head_medicare_eligible,7,llm_error,age_disability,False,,"Under age 65, a disability flag alone does not make someone eligible for Medicare. The disability route requires 24 months of Social Security Disability Insurance (SSDI) entitlement, or end-stage renal disease or ALS. The household reports no SSDI income, and the head earns $41,200 in wages, far above substantial gainful activity. So the only pathway PolicyEngine can apply is the age-65 test, and a 26-year-old fails it. Every model treated the 'is disabled' flag as enough for Medicare and skipped the SSDI-duration requirement."
-us,scenario_102,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee FICA applies to each spouse's full listed gross wages: 7.65% of $41,200 ($3,151.80) plus 7.65% of $46,000 ($3,519.00) equals $6,670.80. Neither spouse reaches the Social Security wage base, combined wages are well below the $250,000 Additional Medicare Tax threshold, and NC has no mandatory employee state payroll tax. The main trap is the $7,989 employer-sponsored insurance premium input. It is not a salary reduction from the listed gross wages, so subtracting it from the spouse's FICA base produces $6,059.64. Most other misses came from models that reached $6,670.80 but submitted a different number, or that used the wrong Medicare rate or the wrong wage total."
+us,scenario_102,payroll_tax,18,llm_error,payroll_tax_base,False,,"The reference applies the 6.2% employee Social Security rate and the 1.45% Medicare rate to each spouse's full listed gross wages: $41,200 and $46,000. That gives 7.65% × $87,200 = $6,670.80. Neither spouse reaches the Social Security wage base, combined wages are far below the $250,000 joint threshold for Additional Medicare Tax, and NC has no employee state payroll tax. The most common mistake was subtracting the spouse's $7,989 employer-sponsored insurance premium from FICA wages as if it were a Section 125 salary reduction, which gives $6,059.64. Most of the other wrong models reached $6,670.80 in their reasoning and then submitted a different number because of arithmetic, rounding or transcription errors."
us,scenario_102,self_employment_tax,2,llm_error,payroll_tax_base,False,,Self-employment tax applies only when net earnings from self-employment reach the filing threshold; the head’s listed $1 produces no liability. There is no minimum $31 self-employment tax or rounding rule that converts this de minimis amount into a positive tax.
us,scenario_102,spouse_medicare_eligible,7,llm_error,age_disability,False,,"Under-65 Medicare eligibility does not come from a bare disability flag. It requires 24 months of Social Security Disability Insurance entitlement, or end-stage renal disease or ALS. The spouse is 26, receives no SSDI (the benefit is unlisted, so it is $0), and has no ESRD or ALS, so only the age-65 test applies and it fails. All seven models used the same made-up shortcut, ""disabled = Medicare eligible,"" and skipped the SSDI 24-month entitlement step."
-us,scenario_102,state_income_tax_before_refundable_credits,26,llm_error,thresholds_rates,False,,"North Carolina's 2026 tax is flat: 3.99% of NC taxable income. For 2026 that rate is set by statute; the revenue triggers in S.L. 2023-134 apply only from 2027. NC taxable income starts from federal AGI of $87,201 ($41,200 + $46,000 wages + $1 self-employment income), minus the $25,500 married-filing-jointly NC standard deduction, which gives $61,701 × 3.99% = $2,461.87. The wrong answers fall into two groups. Thirteen models used a stale or invented rate (4.0%, 4.25%, 4.49%, or 4.5%) and/or the wrong standard deduction ($21,500, $23,400, $23,850, $26,400, $30,000, or none). Eight models subtracted the spouse's $7,989 employer-sponsored insurance premium from wages as a pre-tax exclusion, which turns AGI into $79,212 and the tax into $2,143.11."
+us,scenario_102,state_income_tax_before_refundable_credits,27,llm_error,thresholds_rates,False,,"North Carolina taxes 2026 income at a flat 3.99%. The base is federal AGI of $87,201 ($87,200 in wages plus $1 of self-employment income) minus the fixed $25,500 joint standard deduction, which gives $61,701 and $2,461.87 of tax. Most models went wrong in one of two ways. Some used an outdated or invented rate (4.25%, 4.5%, 4%, 4.49%) or a wrong standard deduction ($21,500, $23,400, $23,850, $26,400, $30,000). Others subtracted the spouse's $7,989 employer-sponsored insurance premium from wages as if it were a pre-tax exclusion, which shrinks AGI to $79,212 even though the stated gross wages already set AGI at $87,201."
us,scenario_102,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_104,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"An income of 0.88 times FPL establishes eligibility only within an applicable MAGI category; this 72-year-old belongs to none of those categories and qualifies through no separate New York Medicaid pathway. Immigration-status eligibility and age do not independently create Medicaid eligibility, and SSI receipt provides no categorical route because SSI is zero."
us,scenario_104,snap,2,llm_error,thresholds_rates,False,,"An elderly SNAP household is exempt from the gross-income test but must still pass the net-income test. Counting Social Security and veterans benefits and applying the allowable standard and medical deductions leaves annual net income around $23,474, or $1,956 per month, above the one-person net-income limit; rent also produces no excess-shelter deduction because it does not exceed half of adjusted income, so the benefit is $0."
@@ -827,14 +826,14 @@ us,scenario_108,federal_income_tax_before_refundable_credits,15,llm_error,taxabl
us,scenario_108,federal_refundable_credits,2,llm_error,age_disability,False,,"The childless EITC has an age-eligibility limit, and an 85-year-old claimant does not qualify even when earnings fall within the credit’s income range. With no qualifying children and no applicable refundable education, child, recovery rebate, or payroll tax credit, total federal refundable credits are $0."
us,scenario_108,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_108,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_108,head_medicaid_eligible,11,llm_error,categorical_eligibility,False,,"Medicaid eligibility requires both a qualifying eligibility category and satisfaction of that category’s financial rules. The head matches no Medicaid category, so the 1.29-times-FPL MAGI level, advanced age, disability flag, low assets, and immigration status do not produce eligibility."
+us,scenario_108,head_medicaid_eligible,12,llm_error,thresholds_rates,False,,"At 85, the head is outside the MAGI adult category, which covers ages 19–64, and their 1.29x FPL MAGI is above Wisconsin's 100% FPL BadgerCare adult limit anyway. That leaves only the SSI-recipient and aged/blind/disabled pathways. The head gets $0 SSI. Their gross income is $30,982 ($7,656 wages + $12,934 SS survivor + $10,392 survivor benefits), about double the FPL. Even the $23,326 of unearned income alone, after the $20 disregard, is far above the aged/blind/disabled income standard. So the engine finds no category (medicaid_category = NONE). Every model treated age 85 plus disability as enough and either called the income 'low' without testing it against a limit or relied on pathways PolicyEngine does not use: medically needy spend-down, the Wisconsin Medicaid Purchase Plan (MAPP), or Indian Health Service (IHS) coverage."
us,scenario_108,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_108,payroll_tax,3,llm_error,other,False,,"Payroll tax equals employee Social Security tax of $474.67 plus employee Medicare tax of $111.01 on $7,656 of wages, for $585.68 total. The decisive step is correctly multiplying the wages by 1.45% and carrying that component into the final sum without transcription or unsupported adjustment."
+us,scenario_108,payroll_tax,4,llm_error,payroll_tax_base,False,,"Only the head's $7,656 of wages is subject to employee FICA. Social Security benefits and survivor benefits are not wages. The correct figure is 6.2% × $7,656 = $474.67 plus 1.45% × $7,656 = $111.01, for $585.68 in total. Wisconsin has no mandatory employee state payroll tax, and Additional Medicare Tax does not apply. Every wrong answer comes from an arithmetic slip, a submitted value that doesn't match the model's own correct calculation, or a missing output. None reflects a different reading of the rules."
us,scenario_108,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_108,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_108,snap,43,llm_error,categorical_eligibility,False,,"This one-person elderly/disabled Wisconsin household is categorically eligible for SNAP through Wisconsin's broad-based categorical eligibility (TANF non-cash). Its $2,581.83 monthly gross income is under the 200% FPL limit ($2,608), and its $270 in assets pass. Categorical eligibility removes the 100% FPL net income test that almost every model used to zero out the benefit. Because 30% of net income ($611) exceeds the $298 maximum allotment, the eligible one-person household gets the FY2026 minimum allotment of $24 a month, or $288 a year. Most models either failed the household on a net or gross income test or set the benefit to $0 when the formula went negative. The two models that found the minimum used the FY2025 figure of $23 instead of $24."
+us,scenario_108,snap,44,llm_error,categorical_eligibility,False,,"Wisconsin FoodShare uses broad-based categorical eligibility (BBCE) through TANF-funded non-cash benefits, with a 200% FPL gross limit. Monthly gross income of $2,581.83 is under that limit (about $2,608), and assets of $270 also pass, so the household is categorically eligible. That waives the 100% FPL net income test. Net income of $2,036 gives a 30% contribution of $611, which is more than the $298 one-person maximum, so the formula benefit is zero. But categorically eligible one- and two-person households still get the minimum allotment, which is $24 a month in FY2026 (8% of $298), or $288 a year. Most models applied the net income test or treated a zero formula benefit as $0. The two $276 answers used the outdated $23 minimum, and several other answers were unsupported benefit amounts."
us,scenario_108,state_income_tax_before_refundable_credits,9,llm_error,state_local_rule,False,,"Every model that answered computed a small positive Wisconsin gross tax, about $110-$165, on roughly $18,048 of Wisconsin income (wages plus the taxable non-Social Security survivor benefits) after the sliding-scale standard deduction and exemptions. None of them then brought that tax to zero with Wisconsin's nonrefundable school property tax credit for renters. That credit is 12% of rent treated as property tax (20% of rent if heat is included, 25% if not, capped at $2,500). On $7,200 of rent it comes to $172.80-$216. That is more than any model's gross tax, so Wisconsin tax before refundable credits is $0. The one model that mentioned the renter credit made it too small and left a balance."
-us,scenario_108,state_refundable_credits,46,prompt_ambiguity,state_local_rule,False,,"The full amount is the Wisconsin homestead credit. Rent-based property taxes are 20% × $7,200 = $1,440. Household income is wages of $7,656 plus nontaxable Social Security of $12,934, which equals $20,590. The separate $10,392 survivor-benefits input is not a homestead income source. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39. Most models added the $10,392 survivor benefits, pushing income to $30,982, above the $24,680 cap, and returned $0. Others skipped the homestead credit, looking only at the child-based Wisconsin EITC, or treated the credit as non-refundable. The few that computed a credit used the wrong income base, the wrong rent percentage, or the wrong phase-out parameters. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)."
+us,scenario_108,state_refundable_credits,47,prompt_ambiguity,state_local_rule,False,,"The whole $271.39 is the refundable Wisconsin Homestead Credit. Household income is $20,590, which is $7,656 in wages plus $12,934 in Social Security survivor benefits; the separate $10,392 in survivor benefits is not part of it. Property taxes are 20% of the $7,200 rent, or $1,440, so the credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39. Most models added the $10,392 to reach $30,982, put the household over the $24,680 income limit and answered $0. Others never applied the Homestead Credit, which is refundable even when no tax is owed. A few found a partial credit but used the wrong income base, the wrong rent percentage or the wrong formula parameters. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)."
us,scenario_109,child1_chip_eligible,6,llm_error,health_coverage,False,,"CHIP is limited to children who do not qualify for Medicaid. At approximately $30,000 of household income, the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility even though the income is below CHIP's upper limit."
us,scenario_109,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Florida's refusal of adult Medicaid expansion does not touch children's coverage: the state's mandatory MAGI pathway for the older-child category (ages 6-18) runs to 133% FPL, and a 9-year-old in a household of five with roughly $21,600 of MAGI sits at 0.72 FPL — barely half the threshold. The only computation required is household MAGI from the $30,000 of self-employment income divided by the family-of-five poverty guideline, then compared with 133% FPL; no asset test, no enrollment fact, and no SSI linkage enters the children's MAGI pathway. Both models returned 0 without ever running that ratio, treating Medicaid eligibility as something the prompt had to assert rather than something the stated income and age determine."
us,scenario_109,child2_chip_eligible,6,llm_error,health_coverage,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, not merely under age 19 and below CHIP's upper income limit. At age 6 and this household income, child2 qualifies for Medicaid under Florida's OLDER_CHILD category, so the Medicaid–CHIP mutual-exclusivity rule makes child2 ineligible for CHIP."
@@ -847,64 +846,64 @@ us,scenario_109,child3_head_start_eligible,1,llm_error,categorical_eligibility,F
us,scenario_109,child3_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The dispositive rule is Florida's age-based INFANT Medicaid category, which covers children under age 1 at MAGI up to 185% FPL — a far higher threshold than the adult standards, and one that applies in a non-expansion state. The household's MAGI is 0.72 FPL, so the only two conditions that matter, age under 1 and income under the infant threshold, are both satisfied. All three models denied eligibility by routing the infant through adult or generic standards: one gave no derivation, one required an SSI/disability-style marker that MAGI categories do not use, and one computed the income test correctly and then overrode it with a nonexistent asset test plus a Medicaid-expansion and enrollment requirement."
us,scenario_109,child3_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is derived from the child's age and household income; it does not require the prompt to explicitly state that the child is WIC-eligible. Child 3 is under age five, and the household's approximately $30,000 annual income is below the 2026 WIC limit of 185% of the federal poverty guideline for a five-person household."
us,scenario_109,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The 2026 married-filing-jointly standard deduction does not reduce this household's taxable income to zero after the deductible half of self-employment tax. The resulting pre-credit income tax is fully offset by $2,792.91 of the $6,600 Child Tax Credit allocated as nonrefundable, and tax after nonrefundable credits is floored at zero rather than becoming negative."
-us,scenario_109,federal_refundable_credits,40,llm_error,thresholds_rates,False,,"The reference is the 2026 three-child EITC maximum of $8,231 plus a refundable CTC of $3,807.09. EITC earned income is $30,000 minus the deductible half of SE tax (about $2,119), which is $27,881. That sits on the plateau, below the MFJ phase-out start. The refundable CTC is 15% of ($27,881 − $2,500), which is under the $1,700-per-child cap of $5,100. Models went wrong in three main ways: (1) using the pre-TCJA $1,000-per-child refundable CTC ($3,000), (2) running the 15% ACTC formula on gross $30,000 or on the 92.35% net-earnings base of $27,705 instead of $27,881, and (3) phasing down an EITC that is actually on the plateau. Several also used EITC maximums other than $8,231, or submitted a number that differs from their own arithmetic."
+us,scenario_109,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"The reference is the full 2026 three-child EITC maximum of $8,231 plus an ACTC of $3,807.09. Earned income is $30,000 minus the deductible half of SE tax ($2,119.43), or $27,880.57. That is on the EITC plateau and below the MFJ phase-out start. The ACTC is 15% × ($27,880.57 − $2,500), which is below the 3 × $1,700 refundable cap under the post-OBBBA CTC. The wrong answers made one or more of four mistakes. Some used the expired-TCJA $1,000-per-child CTC with a $3,000 threshold. Some measured earned income as gross $30,000 or as net SE earnings of $27,705 instead of SE profit minus half of SE tax. Some phased out the EITC when it should not have been. Others used a wrong 2026 EITC maximum."
us,scenario_109,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The FREE tier here is over-determined: the household's $8,020.55 SNAP benefit confers categorical free-meal eligibility outright, and independently $30,000 for a five-person household is 78% of the federal poverty guideline, far under the 130% free-meal cutoff. The only fact requiring inference is school attendance — never stated explicitly — which follows from the children aged 9 and 6 being K-12 students, generating $7.15 per child per day net of the $0.87 paid portion for $2,261.92 annually. The trap is the instruction to treat unlisted statuses as false: a model that applies it to school enrollment erases both students and skips both qualifying tests. Both models defaulted to 0 on that reading rather than deriving the students from age and then applying either the SNAP categorical pathway or the 130% FPG income test."
us,scenario_109,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Low MAGI alone does not establish adult Medicaid eligibility in Florida. The age-30 head is not a dependent, receives no SSI, and fits none of Florida's active Medicaid eligibility categories, so the category is NONE despite MAGI income of 0.72 times FPL."
us,scenario_109,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is individual, not household-wide: only a pregnant, postpartum, or breastfeeding woman, an infant, or a child under age five can qualify. The infant satisfies that requirement, but the 30-year-old Head has no listed pregnancy, postpartum, or breastfeeding status, so the Head remains ineligible despite the household meeting the income test."
us,scenario_109,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price and free school meals are mutually exclusive output tiers in this calculation. The household’s 0.78 school-meal FPG ratio and categorical eligibility place it in the superior FREE tier, so it receives positive free-meal support and no reduced-price support."
us,scenario_109,self_employment_tax,6,llm_error,other,False,,"The Schedule SE computation here is fully determined: net earnings from self-employment are 92.35% of $30,000 = $27,705, entirely below the Social Security wage base, so the tax is $27,705 × 15.3% = $4,238.865, which PolicyEngine reports as $4,238.86. Every model named the correct base fraction, the correct 15.3% combined rate, and correctly noted the wage base is not binding — none missed a rule. The separation is purely in executing the product: each model reported a whole-dollar figure that is $2 to $6 away from $4,238.87, whereas correct rounding of the true product to whole dollars gives $4,239. The trap is arithmetic precision on 0.9235 × 0.153 × $30,000, not any element of the SE tax rule."
-us,scenario_109,snap,41,llm_error,thresholds_rates,False,,"This is a simple SNAP case with no shelter deduction. Start with $2,500 of monthly self-employment income. Subtract the $500 earned income deduction (20%) and the $261 FY2026 standard deduction for five people, which leaves $1,739. The 30% contribution is $521.70, and SNAP rounds it up to $522. The FY2026 maximum allotment for five is $1,183, so the benefit is $661 a month, or $7,932 a year. Rent of $800 is below half of adjusted income ($869.50) and no utility costs are listed, so there is no excess shelter deduction. Most wrong answers land within about $600 a year of $7,932. The cause is using stale or guessed FY2026 figures instead of the exact $261 standard deduction and $1,183 maximum, or skipping the round-up of the 30% contribution, which gives $7,935.60 or $7,936. The rest made bigger errors: they invented shelter or utility deductions, subtracted self-employment tax, miscounted household size, or wrongly found the household ineligible. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))."
+us,scenario_109,snap,42,llm_error,thresholds_rates,False,,"The correct SNAP calculation counts all $2,500 of monthly self-employment receipts, with no SE-tax reduction and no business costs listed. It subtracts the 20% earned income deduction ($500) and the FY2026 standard deduction for five people ($261), giving $1,739. The $800 rent is below half of that amount ($869.50), and no utility costs are listed, so there is no excess shelter deduction. Thirty percent of net income ($521.70) is rounded up to $522 and subtracted from the FY2026 five-person maximum of $1,183, giving $661 a month, or $7,932 a year. Most wrong answers came from using FY2025 or guessed figures for the standard deduction and maximum benefit ($254/$1,155 and others), skipping the round-up to a whole dollar, adding a shelter deduction or utility allowance that doesn't apply, or miscounting the household or eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))."
us,scenario_109,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Florida’s non-expansion Medicaid program does not provide a general low-income pathway for a nondisabled 29-year-old adult. The spouse qualifies through no categorical pathway: household MAGI exceeds Florida’s parent/caretaker limit, and the presence of an infant does not establish pregnancy or postpartum status."
us,scenario_109,spouse_wic_eligible,8,llm_error,categorical_eligibility,False,,"WIC eligibility for an adult requires the adult herself to be pregnant, breastfeeding, or within the applicable postpartum period; merely living with an infant does not establish any of those statuses. The prompt makes every unlisted status false, so the spouse is not pregnant, postpartum, or breastfeeding and fails categorical eligibility regardless of household income."
-us,scenario_110,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine leaves estate_income out of IRS gross income, while still treating it as qualified business income for a $1,264.59 §199A deduction (20% of $6,323). Its AGI of $163,421.64 equals wages $95,755.10 + capital gains $30,869.77 + interest $21,301 + dividends $15,694.12 − $198.35, with no estate income. This contradicts IRC §61(a)(15) and §662(a), which include estate and trust distributions in gross income.","The reference builds AGI of $163,421.64 from wages net of the 401(k), capital gains, interest and dividends, less a $198.35 IRA deduction. It leaves the $6,323 estate income out of AGI but deducts 20% of it ($1,264.59) as QBI, alongside the $16,100 standard deduction and the $387 nonitemizer charitable deduction. That puts ordinary taxable income at $104,785.81, entirely within the 22% bracket. The most common answer ($25,841) includes estate income (AGI $169,943), disallows the IRA deduction, and skips the charitable and QBI deductions, which puts $7,259 of ordinary income in the 24% bracket. The other misses come from assuming the TCJA expired (personal exemptions, 2%-floor miscellaneous deductions, 15/25% brackets), outdated standard-deduction and bracket figures, overtime deductions built on an overtime premium that was never listed, subtracting ESI premiums from wages, and final values that contradict the models' own arithmetic. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 23,897.44; the corrected value is 25,747.77. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))."
+us,scenario_110,federal_income_tax_before_refundable_credits,45,llm_error,taxable_income_or_deductions,False,,"Under OBBBA's permanent 2026 rules, a single filer gets the $16,100 standard deduction plus the new non-itemizer charitable cash deduction (here $387). That leaves taxable income of $153,455.47. Of this, $112,571.70 is taxed at ordinary rates under the 2026 brackets (10%/12,400, 12%/50,400, 22%/105,700, 24%), giving $19,615.21, and $40,883.77 of qualified dividends and LTCG is taxed at 15%, giving $6,132.56. The largest cluster of answers ($25,841) got every step right except the $387 charitable deduction. Most other wrong answers assumed the TCJA had expired, with personal exemptions, 2%-floor miscellaneous deductions and the old brackets. Others excluded employer-paid premiums from wages, invented an overtime deduction from the hours worked, allowed a traditional IRA deduction that phases out at this income, or used outdated brackets and standard-deduction amounts."
us,scenario_110,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_110,payroll_tax,15,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 7.65% of the full $100,000 of gross wages: $6,200 of Social Security tax plus $1,450 of Medicare tax, for $7,650. Wages are below the Social Security wage base and below the $200,000 Additional Medicare Tax threshold, and Ohio has no mandatory employee state payroll tax. The main mistake, made by nine models, was treating the $6,889 employer-sponsored insurance premium as a Section 125 pre-tax salary reduction. The prompt lists no cafeteria plan, and PolicyEngine does not take that premium out of FICA wages. One model subtracted the traditional 401(k) deferral, which stays subject to FICA. Others added an Additional Medicare Tax that does not apply, reported a fabricated $0, or submitted a number that contradicted their own explanation."
+us,scenario_110,payroll_tax,16,llm_error,payroll_tax_base,False,,"The correct employee payroll tax applies 6.2% Social Security tax ($6,200) and 1.45% Medicare tax ($1,450) to the full $100,000 of listed gross wages. That gives $7,650, with no Additional Medicare Tax because wages are well below $200,000, and Ohio has no employee state payroll tax. The main mistake was shrinking the FICA wage base. Most models subtracted the $6,889 employer-sponsored insurance premium as if it were a Section 125 salary reduction, but the facts never say it is. One model subtracted the traditional 401(k) deferral, which does not reduce FICA wages. Other models applied the Additional Medicare Tax to non-wage income, or submitted numbers that contradicted their own correct $7,650 calculation."
us,scenario_110,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_110,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"For 2026, Ohio taxes income above $26,050 at a single 2.75% rate plus a fixed $332 base, and there is no 3.5% bracket. At MAGI above $80,000 the personal exemption is $1,900, and Ohio has no standard or itemized deduction. Ohio AGI is $163,421.64 because the $6,889 of pre-tax employer health premiums and the $4,245 401(k) deferral come out of wages. That leaves $161,521.64 of taxable income, and $332 + 2.75% × $135,471.64 = $4,057.47. Most models either dropped the $332 base (flat 2.75% above $26,050) or used the old 2.75%/3.5% schedule, and most also left the employer premiums in AGI (about $169,900), skipped or mis-tiered the exemption, or subtracted deductions Ohio does not allow. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,057.47; the corrected value is 4,236.81. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))."
+us,scenario_110,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Ohio AGI equals federal AGI of $169,942.94. That figure counts wages net only of the traditional 401(k) deferral. The $198 IRA deduction is fully phased out because the head has workplace coverage and high income, and employer insurance premiums are not subtracted. The $1,900 personal exemption for income above $80k leaves $168,042.94 of taxable income. The 2026 Ohio schedule taxes income above $26,050 at a flat 2.75% plus a fixed $332 base: $332 + 0.0275 × $141,992.94 = $4,236.81. No nonrefundable credit applies. Most models either left out the $332 base (landing near $3,905), kept the repealed 3.5% bracket above $100,000 (landing near $4,400–$4,500), or changed AGI by deducting the phased-out IRA, subtracting the $6,889 employer insurance premiums, adding back the $954 muni interest, or skipping or misstating the $1,900 exemption."
us,scenario_110,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_111,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The IRA distribution produces $18,240 of adjusted gross income, while the Social Security retirement benefit is not taxable under the joint-return provisional-income thresholds. The applicable standard deduction eliminates taxable income, so the tax computation yields $0; both models expressly derived zero taxable income and zero liability but then submitted unsupported positive amounts."
-us,scenario_111,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Age, blindness, disability, and low assets do not by themselves establish Medicaid eligibility; the applicant must qualify through a specific Washington Medicaid category and satisfy that pathway’s rules. The head qualifies through no pathway: medicaid_category is NONE, SSI receipt is zero, and the 2.02-FPL MAGI exceeds the applicable MAGI-category limits."
+us,scenario_111,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"At 72, the head is outside the ACA expansion-adult category, which covers ages 19–64 up to 138% FPL. Their MAGI is also about 202% FPL, so that pathway fails twice. The head gets no SSI, so the SSI-recipient route is closed. The aged/blind/disabled route has an income limit near the SSI/100% FPL level, roughly $1,000–$1,300 a month. The head's unearned income is about $3,637 a month, so it is far over that limit even after the $20 general disregard. All four models treated being aged, blind or disabled plus low assets as enough to qualify and skipped the income test, which leaves medicaid_category = NONE."
us,scenario_111,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits or taxable IRA distributions. Because neither spouse has wages or other employee compensation, the employee Social Security, Medicare, Additional Medicare, and mandatory Washington employee payroll-tax bases are all zero, yielding $0."
us,scenario_111,snap,1,llm_error,categorical_eligibility,False,,"Washington's 200% gross-income screening rule does not itself establish categorical SNAP eligibility or guarantee the minimum allotment. This two-person household's approximately $43,640 annual income disqualifies it, so the minimum-benefit rule never applies and annual SNAP equals $0."
-us,scenario_111,spouse_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Blindness and disability open non-MAGI Medicaid pathways but do not establish eligibility by themselves; each pathway retains its own income, categorical, and sometimes resource requirements. The spouse qualifies through none of those pathways, and the MAGI adult pathway also fails because the applicable income level is 2.02 times FPL, above Washington’s 138% FPL expansion threshold."
+us,scenario_111,spouse_medicaid_eligible,9,llm_error,categorical_eligibility,False,,"The spouse is 49, so their only Medicaid routes are the MAGI adult-expansion group and the blind/disabled pathway. For Medicaid, MAGI adds back non-taxable Social Security. The tax unit's MAGI is therefore the head's full $25,400 Social Security plus $18,240 in IRA distributions, or $43,640. That is 2.02x FPL, far above the 138% expansion limit. The blind/disabled pathway in PolicyEngine is not open-ended. It depends on SSI receipt or an SSI-related income test that counts the couple's combined income. The spouse gets $0 SSI because the head's income is deemed to them and wipes out the couple's SSI benefit. With no category reached, the engine returns medicaid_category NONE. Every model either treated blindness/disability as enough on its own, ignored spousal income, or left Social Security out of MAGI."
us,scenario_111,spouse_medicare_eligible,10,llm_error,age_disability,False,,"Under age 65, a person qualifies for Medicare through disability only after 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), or through the ESRD or ALS pathways. PolicyEngine models the same rule: the age-65 test or at least 24 months of Social Security disability receipt. The spouse is 49 and flagged disabled, but the household lists no Social Security disability income, no months of SSDI receipt, and no ESRD or ALS, so the spouse fails every pathway. All ten models treated the disability flag alone as enough for Medicare. They skipped the separate requirement of 24 months of SSDI entitlement."
us,scenario_112,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The trap is the $13,000 ""financial assistance"" line: in PolicyEngine `financial_assistance` is a non-taxable cash receipt (counted for benefit means-tests, never in gross income or AGI), so the only taxable items here are $5,923 of wages and $1,920 of farm rent income, reduced by the $3,000 capital-loss limitation on the -$9,944 long-term loss, for AGI of roughly $4,843. That sits far below the 2026 single standard deduction (~$16,100), leaving taxable income of $0, no NIIT (net investment income is a $3,000 loss), and no add-on liabilities, hence $0. Both models handled the $3,000 capital-loss cap and the standard deduction correctly but pushed the $13,000 gift/assistance through as ordinary income, manufacturing $1,743 of taxable income and a $174.30 bracket tax. Excluding the non-taxable transfer collapses the entire computation to zero."
-us,scenario_112,federal_refundable_credits,21,llm_error,credit_phaseout,False,,"The case turns on one computation: the childless EITC on the phase-in ramp, 7.65% of the head's $5,923 in wages = $453.10. Farm rent is neither earned income nor enough investment income to trip the ~$12,200 disqualification limit, AGI is $4,843 (wages + $1,920 farm rent − the $3,000 capital-loss cap) and sits far below the ~$10,600 single phase-out start, the head is 31 and so inside the 25-64 childless age band, and the $13,000 financial assistance is not taxable income. The wrong answers cluster into four shortcuts: subtracting the $8,389 employer-sponsored insurance premiums from wages to zero out earned income; applying the 7.65% rate to AGI or to a MAGI inflated by the nontaxable financial assistance and declaring the credit phased out; treating ""no qualifying children,"" negative net income, or zero tax liability as automatically zeroing a refundable credit; and deriving $453 correctly and then submitting a different number."
+us,scenario_112,federal_refundable_credits,22,llm_error,categorical_eligibility,False,,"The head is 31, has no qualifying children, and has $5,923 of wage earnings, so they qualify for the childless EITC. That credit phases in at 7.65% of earned income starting from the first dollar: 7.65% × $5,923 ≈ $453.10. AGI is $5,923 + $1,920 − $3,000 (the capital-loss cap) = $4,843, which is below the phase-out start. The $1,920 of farm rent is far below the investment-income limit. No other refundable credit applies. Most models went wrong in one of three ways: they wrongly ruled out the childless EITC at low earnings, they subtracted the $8,389 employer-insurance premium from wages to get zero earned income, or they worked out about $453 and then changed it to some other number without a valid reason."
us,scenario_112,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Texas does not provide MAGI Medicaid eligibility to a nondependent, childless 31-year-old merely because income is below an income threshold. The head meets the immigration requirement but belongs to none of Texas's active Medicaid categories, so the low 0.30-FPL MAGI level does not create eligibility."
us,scenario_112,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying demographic category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 31-year-old head has no listed qualifying status, and unlisted statuses are false, so passing the income test does not establish WIC eligibility."
us,scenario_112,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employer-sponsored insurance premiums are not automatically employee pre-tax cafeteria-plan deductions from FICA wages; the listed $5,923 remains subject to employee Social Security and Medicare taxes. PolicyEngine computes $367.22 of Social Security tax and $85.88 of Medicare tax, totaling $453.10, with no Additional Medicare Tax or mandatory Texas employee payroll tax."
us,scenario_112,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to positive net earnings from self-employment, after determining the applicable self-employment tax base. The household’s relevant net income is approximately -$2,101, so there are no positive net self-employment earnings and the tax base is zero; treating the $1,920 farm rent entry as a standalone positive self-employment tax base produces the wrong liability."
-us,scenario_112,snap,46,prompt_ambiguity,categorical_eligibility,False,"PolicyEngine's SNAP earned and unearned income source lists leave out the farm_rent_income variable. Its gross income of $1,576.90 equals ($5,923 + $13,000)/12, so the $1,920 of farm rent is excluded, although 7 CFR 273.9(b)(2)(ii) counts net income from rental property (and 273.9(b)(1) counts it as earned income when it is actively managed). Counting the farm rent raises net income to about $1,429/month, above the $1,304 net limit, which makes SNAP $0.","The reference counts only wages ($493.58/month) and the $13,000 of financial assistance ($1,083.33/month) as SNAP income, for $1,576.92/month; it leaves out the $1,920 of farm rent. After the 20% earned-income deduction and the $209 standard deduction, net income is $1,269.19. That is just under the $1,304.17 limit (100% of the poverty guideline), so the household passes the net income test. Because 30% of net income ($380.76) is more than the $298 maximum allotment, this one-person household gets only the minimum allotment of about $23.84/month. Most models added the $160/month of farm rent, which pushed net income to about $1,429 (above the net limit) or applied a 130% gross test despite Texas's 165% broad-based categorical eligibility (BBCE) limit, and answered $0. A few left out the financial assistance and answered about $194/month. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)."
-us,scenario_114,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The reference follows OBBBA law for 2026. AGI is $106,572.56, with 85% of Social Security taxable. Itemized deductions total $24,787.23: charitable $10,913.78 after the new 0.5%-of-AGI floor; SALT $9,431.59, the $4,064.29 real estate tax plus $5,367.30 that the engine estimates as Virginia income tax withheld (Virginia's rates on federal AGI less Virginia's $8,750 standard deduction; the prompt lists no state income tax withheld or paid); and medical $4,441.86. The engine counts $12,434.80 of medical expenses: the $10,000 of other medical expenses plus a $2,434.80 Medicare Part B premium ($202.90 a month) that it adds because it treats the 69-year-old as enrolled in Medicare, which the prompt does not state. It does not count the $2,000 of over-the-counter health expenses. The deduction is the amount above 7.5% of AGI ($7,992.94). The $24,787.23 total beats the $18,150 aged standard deduction plus the $1,000 non-itemizer charitable deduction. Itemizers still take the $4,105.65 phased senior deduction and the $4,872.35 QBI deduction, leaving $72,807.33 taxed at the permanent TCJA brackets (10% to $12,400, 12% to $50,400, then 22%) for $10,729.61. The wrong answers fall into five groups: (1) applying expired pre-TCJA law (personal exemptions, 15%/25% brackets, no QBI); (2) omitting the senior deduction; (3) itemizing less than the engine by putting less state income tax into SALT than the engine's $5,367.30 estimate (none, or the model's own Virginia tax) or by leaving out the Part B premium; in the engine either omission alone still leaves itemizing ahead, and only leaving out both the state income tax and the premium makes the standard-deduction path ($11,969.80) the result, which kimi-k3 ($11,969.85) and gpt-5.6-sol ($11,970) answered, while gpt-6-astra ($11,642.20) and gpt-6.1-sol ($11,642.09) match the engine without the premium and with the engine's own Virginia tax on that reading ($3,654.15) in SALT ($11,642.16); (4) imputing mortgage interest or self-employment tax that the facts do not support; and (5) reporting numbers unrelated to their own arithmetic. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
+us,scenario_112,snap,47,prompt_ambiguity,categorical_eligibility,False,,"PolicyEngine counts wages ($493.58/mo) and the $13,000 financial assistance ($1,083.33/mo) but not farm rent. That gives gross income of $1,576.90 (121% FPL, inside Texas's 165% FPL BBCE limit). After the 20% earned-income deduction and the $209 standard deduction, net income is $1,269.19 (97% FPL). The household is categorically eligible through TANF non-cash BBCE. Thirty percent of net income ($380.76) exceeds the $298 maximum allotment, so the formula benefit is zero, but an eligible one-person household still gets the minimum allotment of about $24/month, or $287.68 for the year. Most models zeroed the benefit in one of three ways: they applied a 130% FPL gross test with no BBCE, they counted farm rent and failed the net test, or they let the formula reach $0 without applying the minimum benefit. Five models made the opposite error: they left out the financial assistance, or used a maximum allotment, and overstated the benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)."
+us,scenario_114,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"AGI is about $106,573 for nearly every model. The answers split on the deduction stack. The reference itemizes $24,787.23: charity after the 0.5%-of-AGI floor, SALT of real estate tax plus about $5,367 of imputed Virginia income tax, and medical expenses above 7.5% of AGI that include a $2,434.80 Medicare Part B premium but exclude over-the-counter costs. It also takes the OBBBA senior deduction phased down to $4,105.65, which itemizers also get, and the $4,872 QBI deduction. Taxable income of $72,807 is then taxed at the permanent 2026 OBBBA single brackets (10/12/22%). Models that applied the TCJA sunset (personal exemption, 10/15/25% brackets, no QBI deduction), left out the senior deduction, understated SALT or medical, or took the standard deduction instead of itemizing all ended up with too much taxable income. A few models also imputed self-employment tax, dropped income, or made arbitrary final adjustments. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
us,scenario_114,federal_refundable_credits,2,llm_error,credit_phaseout,False,,"EITC phaseout uses adjusted gross income or earned income, whichever is greater, rather than wages alone. This filer’s partnership/S-corp income, taxable retirement distributions, pension, taxable Social Security, and interest raise AGI far above the childless EITC limit, so every refundable credit component is zero."
us,scenario_114,payroll_tax,4,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes employee Social Security and employee Medicare taxes on the $22,700 of wages, while self-employment tax is reported separately and Virginia adds no mandatory employee payroll tax here. The stated components sum to $1,407.40 + $329.15 = $1,736.55; arbitrary adjustments, duplication, and invented state payroll taxes produce the wrong answers."
us,scenario_114,self_employment_tax,18,llm_error,payroll_tax_base,False,,"The household's only business-type input is $24,362 of partnership or S-corp income, and that amount is not net earnings from self-employment. S-corp pass-through income is never subject to SE tax. A partnership distributive share counts only for a general partner, and the prompt says any unlisted status is false, so no general-partner status can be assumed. With zero self-employment income, SE tax is $0. Every substantive wrong answer made the same mistake: each model counted the partnership/S-corp amount as SE earnings, then applied 15.3% to 92.35% of it, which gives about $3,442. Some models then added further arithmetic errors on top of that."
-us,scenario_114,state_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"Virginia taxable income here starts from federal AGI ($106,572.56) minus the $19,828.80 Social Security subtraction, giving VA AGI of $86,743.77. The age deduction is zero because a single filer's AFAGI of about $86,744 is more than $12,000 over the $50,000 threshold. From there, the engine applies Virginia itemized deductions of $19,419.94, because the filer itemizes federally in the engine: the federal itemized deductions other than SALT, which are $10,913.78 of charitable gifts after the 0.5%-of-AGI floor and a $4,441.86 medical deduction, plus the $4,064.29 real estate tax, with state income tax left out. The medical deduction counts the $10,000 of other medical expenses and a $2,434.80 Medicare Part B premium that the engine adds because it treats the 69-year-old as enrolled in Medicare, which the prompt does not state; it leaves out the $2,000 of over-the-counter expenses. Exemptions of $930 personal and $800 age-65 are also taken as deductions from income. That leaves $65,593.83, taxed at $720 plus 5.75% of the amount over $17,000 = $3,514.15. The wrong answers split five ways: (1) using the VA standard deduction instead of itemizing (the engine does the same, for $4,127.67, only when it counts neither the Part B premium nor the $5,367.30 estimate of Virginia income tax withheld that it puts in federal SALT, which the prompt also does not list); (2) granting the phased-out $12,000 age deduction; (3) dropping the $800 aged exemption or treating exemptions as tax credits; (4) building itemized deductions without the Part B premium: $16,985 with the charitable floor (gpt-6-astra, $3,654.16, and gpt-6.1-sol, $3,654.66, within a dollar of the engine's $3,654.15 without the premium) or $17,518 without the floor, and, by counting the $2,000 of over-the-counter expenses in its place, $18,985 with the floor or $19,518 without it; (5) skipping the Social Security subtraction. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
+us,scenario_114,state_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The Virginia tax is 720 + 5.75% × ($65,593.83 − $17,000) = $3,514.15. Virginia taxable income is the $86,743.77 VA AGI, after the Social Security subtraction, minus two items. The first is $19,419.94 of itemized deductions: charitable gifts of $10,913.78 after the 2026 federal floor of 0.5% of AGI, a medical deduction of $4,441.86, and real estate taxes of $4,064.29. The medical figure is the $10,000 of medical expenses plus the $2,434.80 Medicare Part B premium, less 7.5% of AGI; the $2,000 of over-the-counter health expenses is not counted. The second is $1,730 of exemptions ($930 personal plus $800 for age 65 or older), and the $12,000 age deduction phases out to zero above the $50,000 single threshold. The wrong answers mostly fell into one of these traps: taking the VA standard deduction instead of itemizing, leaving out the Part B premium or counting the over-the-counter expenses as medical, skipping the charitable floor, dropping the $800 age exemption, wrongly allowing the age deduction, or not subtracting Social Security. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
us,scenario_114,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_115,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and a reported disability do not by themselves place the head in an Alabama Medicaid eligibility category. PolicyEngine assigns medicaid_category = NONE; the 1.47 FPL MAGI amount therefore cannot establish eligibility through a MAGI category, and immigration eligibility alone is insufficient."
us,scenario_115,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_115,snap,5,llm_error,thresholds_rates,False,,"SNAP eligibility uses approximately $23,442 of annual household income here: $18,708 of Social Security plus $4,734 of tax-exempt interest. That income exceeds PolicyEngine’s applicable one-person SNAP eligibility limit, so the calculation stops at ineligibility and returns $0; deductions and minimum-allotment rules do not create a benefit for an ineligible household."
-us,scenario_115,state_income_tax_before_refundable_credits,45,llm_error,state_local_rule,False,,"Alabama exempts Social Security, but it does not follow the federal exclusion for tax-exempt interest. Only interest on Alabama and U.S. obligations is exempt, and PolicyEngine counts the full $4,734 as Alabama AGI. The single filer gets a $3,000 standard deduction and a $1,500 personal exemption, $4,500 in total. There is no age-65 add-on. That leaves $234 of taxable income, and 2% of $234 is $4.68. Every model either left the interest out entirely, started from a federal AGI of $0 with no add-back, or added the interest back and then overstated the deductions so that they covered all $4,734."
+us,scenario_115,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Alabama follows its own income definition. It adds federally tax-exempt interest back into Alabama gross income because Alabama taxes interest on other states' municipal obligations. That puts this filer's Alabama AGI at $4,734, while the $18,708 of Social Security stays exempt. Alabama's single standard deduction is $3,000 at this income level, with no age-65 or disability add-on, and the personal exemption is $1,500, for $4,500 total. That leaves $234 of taxable income, taxed at 2% to give $4.68. Every model either dropped the interest from the Alabama tax base or overstated the deductions so they absorbed the full $4,734."
us,scenario_115,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_116,snap,1,llm_error,categorical_eligibility,False,,"SNAP’s minimum allotment applies only after a household passes the program’s eligibility tests; it cannot override failure of the gross-income test. At $52,800 of annual household income, this household exceeds the applicable 2026 gross-income limit, so the computation stops with no SNAP benefit."
-us,scenario_116,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish Florida Medicaid eligibility. The spouse receives no qualifying Medicaid category in the engine; with medicaid_category = NONE, the zero MAGI income level and bank assets cannot produce eligibility."
+us,scenario_116,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Florida did not expand Medicaid, so a childless, non-pregnant 42-year-old has no MAGI adult or parent category, even with MAGI at 0% FPL (veterans benefits are left out of MAGI). Disability alone does not create eligibility. The disabled pathway requires either receiving SSI or passing Florida's aged/disabled income test. The spouse's $52,800 in veterans benefits counts as unearned income under SSI-related rules, which pushes SSI to $0 and puts the spouse far above the aged/disabled income limit, so medicaid_category is NONE. All three models treated disability status as enough to qualify and never applied the income test that comes with the disabled pathway."
us,scenario_116,spouse_medicare_eligible,4,llm_error,age_disability,False,,"Under age 65, Medicare eligibility comes only from 24 months of Social Security Disability Insurance (Title II) entitlement or from ESRD/ALS. A disability flag on its own does not confer it. The spouse is 42 and receives $52,800 in veterans benefits, which are VA payments and not SSDI. With SSDI income and SSDI months both unlisted, they are 0 under the prompt's rules, so the age test and the SSDI-duration test both fail. All four models treated ""is disabled"" (plus, for one model, VA benefits) as enough for disability-based Medicare, and so answered Yes."
us,scenario_117,child3_early_head_start_eligible,5,llm_error,categorical_eligibility,False,,"Early Head Start eligibility requires both an age-qualified child and satisfaction of the program's low-income eligibility rules; being under age 3 is not independently sufficient. Although Child 3 meets the age condition, household income of approximately $250,733 is far above the applicable poverty-based income limit, so the income test produces no eligibility."
us,scenario_117,child3_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Age alone does not establish Head Start eligibility; the child must also qualify through an applicable eligibility pathway, principally the program's low-income criteria in this case. This household's approximately $250,733 income exceeds the relevant poverty-based threshold, so Child 3 is not eligible."
us,scenario_117,child3_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC child eligibility requires both categorical eligibility and financial eligibility; being age 2 satisfies only the child-category requirement. The household's approximately $250,733 annual income exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household, so Child 3 is not eligible."
-us,scenario_117,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"Under post-2025 law, start from $250,733 AGI and subtract three deductions: the $32,200 joint standard deduction, the $25,000 qualified-overtime deduction, and the $2,000 non-itemizer charitable deduction. The $35,525 FLSA premium is capped at $25,000 for joint filers, with no phaseout below $300,000 MAGI. That leaves $191,533 of taxable income and $31,561 of bracket tax, less the full $6,600 CTC ($2,200 × 3 children under 17). The CDCC is zero because the head has no earned income. The main trap was the overtime deduction: models omitted it, cut it to the $12,500 single cap, phased it out from the $150,000 single threshold, or took the full premium with no cap. Other errors added to it: TCJA-sunset rate schedules, mortgage interest invented from a bare balance, and a CDCC that the head's zero earnings rule out. Models that came close got the overtime deduction right but itemized instead of taking the standard deduction plus the $2,000 non-itemizer charitable deduction, or claimed a CDCC. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_117,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The 2026 return for this married-filing-jointly household uses these rules: AGI of $250,733.36 (wages plus the taxable state refund minus the capital loss), and the full $25,000 OBBBA qualified-overtime deduction, because the joint phaseout starts at $300,000 MAGI. It takes the $32,200 standard deduction plus the new $2,000 non-itemizer charitable deduction, which together ($34,200) beat itemizing. That gives taxable income of $191,533.36 and $31,561.34 of tax under the permanent TCJA brackets. The household then subtracts $6,600 of CTC ($2,200 × 3 children under 17). There is no CDCC because Head has zero earned income. Models went wrong in five main ways: they left out the overtime deduction or capped or phased it out incorrectly, applied TCJA-sunset law, assumed mortgage interest from the loan balance, missed the $2,000 non-itemizer charitable deduction, or claimed a CDCC that the zero-earner spouse blocks. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_117,federal_refundable_credits,5,llm_error,other,False,,"The refundable CTC (ACTC) is capped at the part of the child tax credit left over after the credit has offset income tax. This couple files jointly with about $250,733 of AGI. That is below the $400,000 CTC phase-out, so the full $2,200-per-child CTC ($6,600) is allowed. Their federal income tax is well over $25,000, which absorbs the whole $6,600 as a nonrefundable credit and leaves nothing to refund. EITC is zero at this income, and no other refundable credit applies, so the total is $0. The wrong answers skipped the tax-liability limit and treated the per-child refundable cap ($1,700 or $1,800), or a made-up fraction of it, as money the family receives automatically."
us,scenario_117,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is individual, not household-wide: living with a two-year-old does not make the 44-year-old head categorically eligible. The household's approximately $250,733 annual income also exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household."
-us,scenario_117,payroll_tax,26,llm_error,payroll_tax_base,False,,"The trap is the 2026 Social Security taxable maximum: $184,500, which caps the employee OASDI tax at $11,439, while the 1.45% Medicare tax runs uncapped on all $248,675 of Spouse wages for $3,605.79, giving $15,044.79. Two secondary traps sit alongside it — the $35,525 FLSA overtime premium is already inside the stated gross wages and must not be added on top, and combined wages fall $1,325 short of the $250,000 married-filing-jointly Additional Medicare Tax threshold, with Arkansas imposing no mandatory employee-side payroll tax. Almost every wrong answer is the correct structure evaluated at a stale or invented wage base (the 2025 $176,100, the 2024 $168,600, or projections between $168,600 and $184,000), and each such answer's shortfall equals 6.2% of the amount by which it understated the taxable maximum. The remaining misses drop the cap entirely and apply a flat 7.65% to full wages ($19,023.64), stack the overtime premium onto wages, invent an Arkansas employee payroll tax, or abandon a correct derivation and submit an unrelated number."
+us,scenario_117,payroll_tax,27,llm_error,thresholds_rates,False,,"Only the spouse has wages ($248,675, which already include overtime). Employee Social Security tax is 6.2% up to the 2026 wage base of $184,500, which comes to $11,439. Medicare is 1.45% of all wages, $3,605.79. Combined wages are under the $250,000 MFJ Additional Medicare threshold, and Arkansas has no employee payroll tax, so the total is $15,044.79. Most models used an outdated or guessed Social Security wage base, such as $168,600 (2024) or $176,100 (2025). Others counted the FLSA overtime premium twice, skipped the SS cap, or submitted a number that does not match their own arithmetic."
us,scenario_117,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_117,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is determined individually and requires both membership in an eligible category—pregnant, postpartum, breastfeeding, infant, or child under five—and satisfaction of the income or adjunctive-eligibility rules. The spouse is not placed in any eligible WIC category by the stated facts, and household income of approximately $250,733 also exceeds the applicable 185% federal poverty guideline."
-us,scenario_117,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Arkansas AGI is $248,144.53, which is the wages less the $530 capital loss. It excludes the $2,589 state refund, and the FLSA overtime premium is already inside gross wages rather than being a separate add-on or a state exclusion. The household itemizes, and its only allowable item is charity: $25,900 minus the 2026 floor of 0.5% of federal AGI ($1,253.67), which leaves $24,646.33. No mortgage interest is allowed because only a loan balance is listed, and medical expenses fall below the floor. Taxable income is $223,498.20, and the engine's 2026 schedule taxes it at $8,189.53, an effective rate of about 3.66%. That is roughly $450 below what a schedule with a 3.9% top rate gives ($8,635). Subtracting $145 in personal credits and the $60 additional tax credit for qualified individuals leaves $7,984.53. The wrong answers fall into four groups: taking the standard deduction instead of itemizing, inventing mortgage interest, using a 3.9%, 4.4% or 4.9% top rate, or leaving out some or all of the $205 in nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"The reference calculation starts from Arkansas AGI of $248,144.53. That figure is wages plus the $530 capital loss. It excludes the $2,589 state refund and keeps the FLSA overtime in income. Arkansas itemized deductions are only the charitable gift after the 2026 floor of 0.5% of AGI: $25,900 minus $1,253.67 equals $24,646.33. There is no mortgage interest because only a loan balance is given, medical costs are below the floor, and employee expenses are below the 2% floor. That leaves taxable income of $223,498.20. The 2026 Arkansas rate schedule produces $8,189.53 of tax on that income. Subtracting $205 of nonrefundable credits ($145 in personal credits plus a $60 additional credit for qualified individuals) gives $7,984.53. The wrong answers split three ways: some skipped the charitable itemization or added imputed mortgage interest, SALT or overtime exclusions; some used old 4.4%/4.7%/4.9% top rates or a high-income schedule that overstates bracket tax; and every model left out the $60 additional credit, with many also omitting or miscounting the $145 personal credits."
us,scenario_117,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_118,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New York automatically provides Medicaid eligibility through the SSI-recipient category. The 74-year-old passes SSI's aged, resource, and income tests, receives $9,368 in annual SSI, and therefore qualifies for Medicaid independently of MAGI pathways."
+us,scenario_118,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The head is 74, blind, and disabled. Their only income is $2,800 a year in Social Security, and they hold $70 in resources. That qualifies them for SSI: countable income after the $20 general exclusion is about $213/month, which is under the federal benefit rate, and the engine pays $9,368 a year. In New York, SSI receipt brings Medicaid eligibility automatically through the SSI-recipient category, and the head also meets NY's aged/disabled income and asset limits. Both models missed that the SSI pathway makes the head categorically eligible. They also failed to check the very low income and assets against the aged/blind/disabled limits."
us,scenario_118,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earned compensation, not Social Security retirement benefits. With no wages, self-employment income, or other covered earnings listed, the employee-side payroll-tax base is zero and payroll tax is $0."
-us,scenario_118,snap,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is 74, blind and disabled, has $233.33/month of Social Security and $70 in assets. That qualifies the head for federal SSI of $780.67/month: the $994 benefit rate less $213.33 of countable Social Security after the $20 exclusion. SSI counts as SNAP unearned income, so gross income is $1,014. After the $209 standard deduction, PolicyEngine applies an uncapped elderly/disabled excess shelter deduction. Shelter costs are about $1,013/month: $599 of mortgage interest and property tax plus New York's standard utility allowance. That leaves net income of about $194, a $58.20 expected contribution, and $298 − $58.20 = $239.80/month for January–September. October–December pays $248.58 under FY2027 parameters, for $2,903.94 in total. The wrong answers fall into three groups. About half the models left out SSI, drove net income to $0 and paid roughly the maximum ($3,500/year). About ten counted SSI but used only mortgage interest and property tax as shelter costs (net ≈ $609, about $1,384/year). The rest denied eligibility outright. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)."
-us,scenario_118,ssi,35,llm_error,state_local_rule,False,,"The ssi output is the federal SSI payment only. It uses the 2026 FBR of $994/month minus countable unearned income of $213.33/month ($233.33 Social Security less the $20 general exclusion, applied monthly), which gives $780.67/month or $9,368/year. New York's state supplement is not part of ssi. Most wrong answers fall into one of three errors: adding the NY SSP (about $87/month, or $1,044/year), using a stale or understated FBR ($943, $967, $982-$993), or applying the $20 exclusion once a year ($2,780 countable) instead of monthly ($2,560). The $0 answers either treated $2,800 of annual Social Security as exceeding the monthly FBR, ignored the listed aged, blind and disabled status, or computed a positive benefit and then submitted zero."
-us,scenario_118,state_refundable_credits,34,llm_error,state_local_rule,False,,"The whole $375 is New York's Real Property Tax Credit (IT-214), which is refundable. A homeowner who pays real property taxes can qualify without NY tax liability, earned income, or rent. From 2025 the credit is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives $375. Most models never considered the RPTC. Others used the wrong cap ($75, $171, or $1,000), used an invented formula, or counted the nonrefundable NY household credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)."
+us,scenario_118,snap,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The question lists $2,800 of Social Security. That is far below the 2026 federal SSI benefit rate of $994/month, so the head also receives SSI of $780.67/month ($994 − ($233.33 − $20)). For SNAP, SSI is countable unearned income, so gross income is $1,014/month, not $233. SSI receipt also makes the household categorically eligible. After the $209 standard deduction, adjusted income is $805. PolicyEngine's uncapped elderly/disabled excess shelter deduction then brings net income down to $194.38: the shelter costs are about $1,013/month, which is the $598.75 of mortgage interest and property tax plus a utility allowance. That gives $298 − $58.20 = $239.80/month for January–September and $248.58/month for October–December under FY2027 values, or $2,903.94 for the year. Most models either dropped SSI from income, which pushed net income to about zero and the benefit toward the maximum, or counted SSI but used only mortgage interest and property tax as shelter costs, giving net income of $608.75 and about $1,384/year. Some answered $0 because they misread eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)."
+us,scenario_118,ssi,36,llm_error,thresholds_rates,False,,"The head is categorically eligible as aged and blind. Countable resources are $70. Countable unearned income is $2,800 minus the $20/month general exclusion ($240/year), which leaves $2,560/year ($213.33/month). The federal SSI payment is the 2026 individual FBR of $994/month minus $213.33, or $780.67/month, which comes to $9,368/year. The `ssi` output is federal SSI only, so no New York State Supplement is added. Models missed this in three main ways: they used a stale or wrong FBR ($943, $967, $983, $990, $991, $993), they added a NY SSP on top of federal SSI, or they applied the $20 exclusion only once per year. Others wrongly found the head ineligible even though $213.33/month of countable income is far below the FBR."
+us,scenario_118,state_refundable_credits,35,llm_error,state_local_rule,False,"PolicyEngine's NY real property tax credit measures household gross income without nontaxable Social Security and SSI. Tax Law §606(e)(1) and Form IT-214 (line 10 Social Security; line 12 public assistance/SSI) include both. That puts this 65+ filer in the ≤$1,000 band with the $375 cap. Counting the $2,800 of Social Security alone gives the $2,001–$3,000 band and a $341 cap, and adding SSI gives the $12,001–$13,000 band and a $171 cap.","The whole amount comes from New York's real property tax credit (IT-214, Tax Law §606(e)). It is a refundable circuit-breaker credit for homeowners and renters with household gross income of $18,000 or less, and it is paid even when the filer owes no tax. For a 74-year-old homeowner paying about $1,634 in property taxes against almost no income, the property taxes are far above the income-based threshold, so the 65+ maximum caps the credit at $375. Most models only checked the EITC and child credits, assumed a refundable credit needs tax liability or rent, or dismissed homeowners. The rest used the under-65 cap, added the nonrefundable household credit, made up a formula, or used a lower income band by counting Social Security and SSI. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)."
us,scenario_119,child1_chip_eligible,17,llm_error,health_coverage,False,,"Child 1 is 14, so the age test passes. Household MAGI of about $52,569 (roughly 192-197% FPL) is also inside Virginia's FAMIS band, so the income test passes too. The child fails CHIP's categorical test instead. The household facts say the child has employer-sponsored insurance, and a CHIP targeted low-income child cannot be covered under a group health plan. Every answering model ran only the age and income-band screen and ignored the child's ESI coverage flag. None of them applied the no-other-coverage requirement, so all returned eligible."
us,scenario_119,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -918,7 +917,7 @@ us,scenario_119,child2_medicaid_eligible,5,llm_error,thresholds_rates,False,,"Th
us,scenario_119,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child2_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,federal_income_tax_before_refundable_credits,21,llm_error,thresholds_rates,False,,"For 2026, OBBBA keeps the TCJA structure. That means a $24,150 head-of-household standard deduction, no personal exemptions, and a $2,200 CTC for each child under 17, so this household's CTC is $4,400. The CDCC rate at this AGI is 35%, giving $546. Tax before credits is $3,056.23. The CDCC and the $200 Saver's credit bring it down to $2,310.23. The nonrefundable CTC is limited only by that remaining liability, so the $4,400 credit wipes it out and the result is $0. The wrong answers came from four mistakes: assuming TCJA expired (personal exemptions and a $1,000-per-child CTC), filing as single, reserving part of the CTC for the refundable ACTC before applying the nonrefundable part, or leaving out credits."
-us,scenario_119,federal_refundable_credits,46,reference_engine_defect,credit_phaseout,False,,"This case has two traps. First, the two-child EITC does not phase out completely here. The 2026 credit is $7,316 and phases out at 21.06% above $23,890 (head of household). The phase-out applies to the greater of AGI ($52,569) and earned income, and earned income is the full $55,000 of wages, so the EITC is $7,316 − 0.2106 × ($55,000 − $23,890) = $764.23. Second, the refundable CTC is the leftover part of the 2026 CTC ($2,200 per child, $4,400 total). Regular tax is about $3,056 on $28,419 of taxable income after the $24,150 head-of-household standard deduction. The 35% CDCC ($546) and the 10% Saver's Credit ($200) reduce it to $2,310.23, and the CTC absorbs only that amount. The remaining $2,089.77 is refundable, which is under the $1,700-per-child cap and the 15%-of-earnings-over-$2,500 limit. Models that zeroed the EITC, used AGI or wages net of the 401(k) as the phase-out base, used $2,000 per child, or skipped the CDCC and Saver's Credit before the CTC all missed $2,854.01. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
+us,scenario_119,federal_refundable_credits,47,reference_engine_defect,credit_phaseout,False,"PolicyEngine's EITC uses the full $55,000 of gross wages as earned income: 7,316 − 0.2106 × (55,000 − 23,890) = 764.23. Its own AGI of $52,569 excludes the $3,087 traditional 401(k) deferral. IRC §32(c)(2)(A)(i), as amended by EGTRRA 2001, counts wages only if they are includible in gross income, and §402(e)(3) deferrals are not. Under that rule the phase-out base is max(AGI $52,569, earned income $51,913) = $52,569, which gives an EITC of $1,276.20 and a total of $3,365.97.","The refundable total is a $2,089.77 ACTC plus a $764.23 EITC. The 2026 OBBBA CTC is $2,200 per child, or $4,400. It first offsets a net liability of $2,310.23: regular tax of $3,056.28 on $28,419 (AGI of $52,569 minus the $24,150 HoH standard deduction), less a $546 CDCC (35% of $1,560) and a $200 Saver's Credit. The remaining $2,089.77 is fully refundable because it is under both the $1,700-per-child cap and the 15%-of-earnings limit. The EITC stays positive because the $7,316 two-child maximum phases out at 21.06% above $23,890 on a $55,000 earnings base. Models missed by zeroing the EITC, assuming tax absorbed the whole CTC, using a $2,000 or $1,000 per-child CTC, applying the old 20% CDCC rate, omitting the Saver's Credit, subtracting employer insurance premiums from AGI, using the wrong filing status, or phasing out the EITC on AGI. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
us,scenario_119,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,head_medicaid_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -930,10 +929,10 @@ us,scenario_119,reduced_price_school_meals_eligible,7,llm_error,taxable_income_o
us,scenario_119,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_119,state_income_tax_before_refundable_credits,46,reference_engine_defect,credit_phaseout,False,"PolicyEngine phases out the federal EITC on $55,000 of gross wages that still include the $3,087 traditional 401(k) deferral ($7,316 − 21.06% × ($55,000 − $23,890) = $764.23 to the cent), but IRC §32(c)(2)(A)(i) counts employee pay as earned income only when it is includible in gross income; the phase-out income should therefore be AGI of $52,568.54, which gives a federal EITC of about $1,276.30, a Virginia 20% nonrefundable EITC of about $255.26, and an output of about $1,756.68.","Virginia taxable income starts from federal AGI ($52,568.54) and subtracts the 2026 $8,750 single/HOH standard deduction, three $930 personal exemptions ($2,790), and the $1,560 child and dependent care expense deduction. That gives $39,468.54 of taxable income and $2,011.94 of bracket tax, and Virginia's 20%-of-federal nonrefundable EITC comes off that before refundable credits. Most models used a stale $8,500 or older standard deduction, skipped the $1,560 childcare deduction, invented Virginia childcare or child credits, reduced AGI by the $6,389 ESI premiums, or dropped the nonrefundable EITC. The models that got both the bracket tax and the EITC step right differ from the reference only in the federal EITC base: they phased it out on AGI, while the reference's $764.23 federal EITC is phased out on $55,000 of gross wages. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
+us,scenario_119,state_income_tax_before_refundable_credits,47,reference_engine_defect,credit_phaseout,False,"The federal EITC of $764.23 matches exactly $7,316 − 21.06% × ($55,000 − $23,890), which means PolicyEngine phased out the EITC on gross wages of $55,000. IRC §32(c)(2)(A)(i) counts compensation as earned income only when it is includible in gross income, so the $3,087 traditional 401(k) deferral is excluded. Phase-out income should therefore be max(AGI $52,569, earned income $51,913) = $52,569, giving a federal EITC of about $1,276, a VA nonrefundable EITC of about $255, and VA tax of about $1,756.7.","The Virginia result takes three subtractions from AGI of $52,568.54: the 2026 standard deduction of $8,750, three $930 personal exemptions ($2,790), and Virginia's child and dependent care expense subtraction ($1,560). That leaves taxable income of $39,468.54 and tax of $2,011.94, which is then reduced by the 20% nonrefundable Virginia EITC. Most models went wrong on the deductions: they used the older $8,500 (or $4,500) standard deduction, left out the $1,560 child-care subtraction (ending at $2,101.67 or $2,116), or subtracted $6,389 of ESI premiums from AGI. Many also skipped the nonrefundable EITC, or invented Virginia child tax credits or child-care credits that do not exist; Virginia handles child care as a deduction, not a credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))."
us,scenario_119,state_refundable_credits,15,llm_error,state_local_rule,False,,"Virginia makes filers choose between a nonrefundable EITC equal to 20% of the federal EITC and a refundable EITC. Both versions cannot be claimed. The household has federal AGI of about $52.6k after the traditional 401(k) and IRA deductions and a federal EITC of about $1.3k. Its Virginia liability is roughly $2,000 (standard deduction and three exemptions, then 5.75% on income above $17k). That liability absorbs the entire nonrefundable credit. PolicyEngine therefore puts the Virginia EITC among the nonrefundable credits that reduce state_income_tax_before_refundable_credits, so va_refundable_credits and state_refundable_credits are $0. Every model that answered treated Virginia's EITC as automatically refundable and put a percentage of the federal EITC into refundable credits, with rates and federal bases that differed from model to model."
us,scenario_119,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 law as made permanent by the OBBBA. AGI is $232,635: wages are reduced only by the $617 traditional 401(k) deferral, the farm rent loss is netted, and the state refund is not included. Itemized deductions total $35,717: SALT of about $13,908, which is the full Connecticut income tax plus property tax under the $40,400 cap; mortgage interest of $2,808, prorated by the $750k acquisition-debt limit; and charitable gifts of $19,001 after the new 0.5%-of-AGI floor. Taxing $196,918 at the 2026 10/12/22/24% single schedule gives $39,858.42, and NIIT of $163.40 on $4,300 of interest and rental income is added. The wrong answers split into four groups: models that assumed a TCJA sunset (personal exemption, pre-TCJA brackets, miscellaneous deductions); models that used the old $10,000 SALT cap or dropped Connecticut income tax; models that subtracted ESI premiums from wages or skipped the charitable floor; and near-misses that omitted NIIT or computed it on a base netted for the farm rent loss. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_120,federal_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"For 2026, the reference uses the OBBBA law, which made the TCJA structure permanent. Single brackets are 10/12/22/24%, with the 24% bracket running from $105,700 to $201,775, and there is no personal exemption and no 2%-floor miscellaneous deduction. AGI is $232,635. The state refund is not included, and the farm rent loss and the $28 IRA deduction are taken above the line. Itemized deductions are $35,717: SALT of $13,908 (CT income tax plus property tax, under the $40,400 cap), mortgage interest of $2,808 (3,744 × 750k/1M), and charity of $19,001 after the 0.5%-of-AGI floor. That gives taxable income of $196,918 and $39,858 of bracket tax, plus $163.40 of NIIT on $4,300 of interest and rental income. Most wrong answers came from assuming the TCJA expired in 2026, using the old $10,000 SALT cap, leaving CT income tax out of SALT, skipping the debt limit or the charitable floor, subtracting ESI premiums from wages, or dropping or shrinking the NIIT. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_120,federal_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -941,12 +940,12 @@ us,scenario_120,head_medicaid_eligible,1,parse_contract_failure,missing_output,F
us,scenario_120,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility applies because the head is age 76, well above the age-65 threshold. Income, assets, disability status, and employer-sponsored insurance do not negate age-based Medicare eligibility."
us,scenario_120,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,payroll_tax,36,llm_error,state_local_rule,False,,"The reference has three parts, all on the full $165,597 of wages: 6.2% Social Security ($10,267.01, since wages are under the 2026 wage base), 1.45% Medicare ($2,401.16), and Connecticut's mandatory 0.5% Paid Leave employee contribution ($827.98). There is no Additional Medicare Tax because wages are under $200,000. The most common mistake was leaving out the CT paid-leave contribution, which gives exactly $12,668.17. The second was subtracting the $7,429 of employer-sponsored insurance premiums (and sometimes 401(k) deferrals) from the payroll-tax base, even though PolicyEngine applies payroll tax to the listed gross wages and traditional 401(k) deferrals are FICA wages under IRC §3121(v)(1)(A)."
+us,scenario_120,payroll_tax,37,llm_error,state_local_rule,False,,"The reference is employee Social Security tax (6.2% × $165,597 = $10,267.01), plus Medicare tax (1.45% × $165,597 = $2,401.16), plus Connecticut's mandatory employee Paid Leave contribution (0.5% × $165,597 = $827.98), for a total of $13,496.16. The wages are below the 2026 Social Security wage base and below the $200,000 Additional Medicare Tax threshold, so the full gross wage is the base for all three taxes. Most models said Connecticut has no employee payroll tax and stopped at $12,668.17. A second group took the $7,429 ESI premium (and sometimes the 401(k) deferrals) out of the wage base, which shrinks every component. The facts give gross wages and never describe the premium as a cafeteria-plan salary reduction, and 401(k) deferrals are FICA wages anyway."
us,scenario_120,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"This filer is above Connecticut's $75,000 single threshold. At that income, Connecticut subtracts the federally taxable Social Security that exceeds 25% of total benefits ($35,615 − $10,475 = $25,140). CT AGI is therefore $232,635 − $25,140 = $207,495, and the personal exemption, pension subtraction, personal credit and property tax credit are all fully phased out. The 2026 schedule (2/4.5/5.5/6/6.5%) gives $11,237.18. Adding the $250 2% phase-out add-back and $430 of benefit recapture ($250 low + $180 middle) gives $11,917.18. The models went wrong in five main ways: they denied any Social Security subtraction at this income, subtracted all taxable benefits, used the pre-2024 3%/5% bottom rates, invented deductions or credits, or dropped part of the $680 add-back and recapture. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 11,917.18; the corrected value is 11,919.06. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_120,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Connecticut starts from federal AGI of $234,414. For a single filer above the $75,000 AGI threshold, Connecticut taxes no more than 25% of Social Security benefits ($10,475), so it subtracts $25,140 of the $35,615 federally taxable amount. It also subtracts the $1,750 state tax refund, leaving CT AGI of $207,524. The 2024+ single schedule (2%/4.5%/5.5%/6%/6.5%) gives $11,239.06 on that amount. Connecticut then adds a $250 phase-out add-back, $250 of low-tax recapture and $180 of middle-tax recapture, for a total of $11,919.06. The personal exemption, personal credit and property tax credit are all zero at this income. The wrong answers mostly fall into a few groups: they skipped the 25%-of-benefits Social Security rule (either no subtraction or a full one), used the old 3%/5% bottom brackets, omitted some or all of the $680 add-back and recapture, or claimed exemptions, deductions or credits that Connecticut does not allow at this income."
us,scenario_120,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_121,federal_income_tax_before_refundable_credits,4,prompt_ambiguity,taxable_income_or_deductions,False,,"Two facts drive the $0: PolicyEngine's `disability_benefits` input is a non-taxable income source excluded from AGI, so AGI is wages only ($25,665), and the household's medical costs ($2,200 non-Medicare-Part-B premiums + $30,000 other medical + $400 over-the-counter = $32,600) generate an itemized medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675. That itemized total exceeds AGI, so taxable income floors at $0 and there is no tax to reduce with nonrefundable credits. Every wrong model either taxed the disability benefits, ignored the medical itemized deduction, or both, leaving a positive 10%/12%-bracket residue. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 875.34. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
@@ -959,12 +958,12 @@ us,scenario_121,reduced_price_school_meals_eligible,1,parse_contract_failure,mis
us,scenario_121,snap,30,llm_error,age_disability,False,,"Every model with an answer let the head's ""is disabled"" flag put the household on SNAP's elderly/disabled track. That track skips the gross income test and allows the uncapped excess medical deduction. But SNAP defines a disabled member by benefit receipt under 7 CFR 271.2: SSI, Social Security disability, or comparable government disability or blindness payments. This head gets no SSI and no Social Security disability, only generic disability benefits, so the household is an ordinary one-person unit. Its gross income is about $3,326 a month ($25,665 wages + $13,044 disability benefits + $1,200 child support, divided by 12), well above South Carolina's gross limit of 130% of the poverty line (about $1,696 a month in FY2026), so SNAP is $0 and the $30,000 in medical costs never matters."
us,scenario_121,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_121,state_income_tax_before_refundable_credits,8,prompt_ambiguity,taxable_income_or_deductions,False,,"South Carolina's tax base starts from federal taxable income, and this filer's deductible medical outlays total $32,600 ($30,000 other medical expenses + $2,200 health insurance premiums excluding Medicare Part B + $400 over-the-counter health expenses). Netted against the 7.5%-of-AGI floor (at most $2,903 on the largest possible AGI of $38,709), that leaves roughly $30,000 of itemized medical deduction — nearly double the $16,100 single standard deduction — which wipes out federal taxable income, and SC's subtraction for the disabled filer's disability income removes anything left, so SC taxable income never reaches the top of the zero-rate bracket and the tax is $0. Every wrong model defaulted to the standard deduction and treated the $30,000 medical line as an irrelevant household fact; several compounded that with invented SC rate schedules and a nonexistent personal exemption. The gap between $0 and every submitted number is entirely the unclaimed itemized medical expense deduction — no bracket or rate correction closes it. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 212.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
-us,scenario_122,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"AGI is $70,646.18: $43,446 of non-Social Security income plus $27,200 of taxable Social Security (the 85% cap binds). Under OBBBA's permanent TCJA structure, 2026 deductions total $27,200. That is the $16,100 basic standard deduction, plus $2,050 each for age and blindness, plus the $6,000 senior deduction (MAGI is under $75,000), plus the new $1,000 charitable deduction for single non-itemizers. Taxable income is therefore $43,446.18. The $1,824 of qualified dividends falls in the 0% band, and the remaining $41,622.18 is taxed at 10%/12%, giving $4,746.66. The wrong answers made one of four errors: they assumed the TCJA sunset (personal exemption, itemizing with miscellaneous deductions, a 15% bracket); they dropped the $6,000 senior deduction; they dropped only the $1,000 non-itemizer charitable deduction, which leaves them exactly $120 high; or they zeroed out the tax with credits that do not exist."
+us,scenario_122,federal_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"AGI is $70,646.18. That is $43,446 of non-Social Security income, after the $1,519 net capital loss, plus the 85% cap of $27,200 in taxable Social Security. Under the OBBBA rules for 2026, the deductions are: the $16,100 standard deduction, plus two $2,050 aged/blind amounts, plus the $6,000 senior deduction (MAGI is under $75,000), plus the new $1,000 charitable deduction for single non-itemizers. That totals $27,200, which leaves $43,446.18 of taxable income. The $1,824 of qualified dividends falls in the 0% band, and the $41,622.18 of ordinary income is taxed at 10%/12%, giving $4,746.66. Wrong answers came from four errors: assuming TCJA expired (personal exemption, 15% bracket, miscellaneous itemized deductions), leaving out the senior deduction or the $1,000 non-itemizer charitable deduction, mishandling the Social Security or dividend income inclusion, or zeroing out a clearly positive liability."
us,scenario_122,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_122,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Age, blindness, and disability do not by themselves establish Medicaid eligibility under Minnesota’s aged, blind, or disabled pathways. The head qualified through no Medicaid category, received no SSI, and had MAGI equal to 4.85 times FPL, so the eligibility result is No."
us,scenario_122,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary earnings, not Social Security benefits, IRA distributions, pensions, dividends, interest, or capital gains. Because the prompt lists no wages and requires unlisted numeric inputs to be zero, the employee payroll-tax base is zero and payroll tax is $0."
us,scenario_122,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_122,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Minnesota's computation starts from federal AGI of $70,646.18. From that it subtracts the full $27,200 of federally taxable Social Security, because AGI is below the single-filer phaseout. It also takes the $4,977.18 nonitemizer charitable subtraction and the $19,300 standard deduction, which includes the age-65 and blind additions and beats itemizing. That leaves $19,169, all taxed at the 5.35% first bracket, for $1,025.54, and no nonrefundable credit applies. Three errors split the wrong answers. Models near $1,300 left out the charitable subtraction. Models near $2,900 to $3,200 skipped or only partly applied the Social Security subtraction and landed in the 6.8% bracket. The $0 answers made up senior or elderly/disabled subtractions or credits, but at this AGI the elderly/disabled subtraction phases out to zero. Models near $1,000 had the right structure but misestimated the deduction amounts, left out the charitable subtraction, added tax-exempt interest, or offset errors with a made-up personal exemption."
+us,scenario_122,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"Minnesota starts from federal AGI of $70,646.18. At this income a single filer gets the full Social Security subtraction of $27,200, the standard deduction with both the age and blindness add-ons ($19,300), and the non-itemizer charitable subtraction ($4,977.18). That leaves $19,169 of taxable income, all taxed in the 5.35% bracket, for $1,025.54. Wrong answers fell into three groups. The highest ones limited or skipped the Social Security subtraction and pushed income into the 6.8% bracket. The $0 answers made up senior or pension subtractions, personal exemptions, or credits that zero out the tax. Most of the near misses left out the charitable subtraction or misjudged the age/blind standard deduction."
us,scenario_122,state_refundable_credits,7,llm_error,credit_phaseout,False,,"The only candidate Minnesota refundable credit is the renter's credit. There are no earnings or children, so the Working Family Credit and Child Tax Credit do not apply. The household's income is about $77,460: AGI plus nontaxable Social Security and tax-exempt interest. At that income the 2026 renter's credit schedule gives nothing, so mn_refundable_credits and state_refundable_credits are both $0. Every model that answered assumed the household falls inside a paying bracket of the renter's credit schedule and granted $1,000-$2,300 from 17% of the $28,235 rent. None of them noticed that the credit is fully phased out at this income."
us,scenario_123,child1_chip_eligible,4,llm_error,thresholds_rates,False,,"Child 1 is 16, so the CHIP age rule (under 19) is met. What decides the case is the income test. The child has to file a return on $45,000 of wages, so that income counts in household MAGI. With the parents' income, household MAGI is about $145,002, which is over 500% FPL for a family of three. That is far above Pennsylvania's CHIP income ceiling (314% FPL), and PolicyEngine's is_chip_eligible applies that ceiling and does not treat the unsubsidized full-cost CHIP buy-in as eligibility. Every model that answered 'Yes' skipped the income ceiling: either it assumed PA's full-cost CHIP makes any income eligible, or it looked only at the child's age."
us,scenario_123,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -972,20 +971,20 @@ us,scenario_123,child1_head_start_eligible,1,parse_contract_failure,missing_outp
us,scenario_123,child1_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_123,federal_income_tax_before_refundable_credits,45,llm_error,household_unit_or_filing_status,False,,"The output is the liability of one tax unit, the joint return. Its AGI is $100,002 because the dependent child's $45,000 of wages is not the parents' income, and no separate child return is added. Under the 2026 OBBBA rules, the parents take the $32,200 standard deduction plus the $2,000 charitable deduction for non-itemizers. That leaves $65,802 of taxable income and $7,400.24 of tax, and the $2,200 CTC brings it to $5,200.24. The models went wrong in five main ways: counting the child's wages in joint AGI ($11,600.44 or higher), adding the child's own $3,220 tax ($8,420.24), using the expired pre-TCJA rules, leaving out the $2,000 charitable deduction, or giving the head an overtime deduction even though the head is not flagged as paid hourly."
+us,scenario_123,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,household_unit_or_filing_status,False,,"The output is the joint return for Head and Spouse. Their AGI is $100,002 (Head's $100,000 wages plus $2 interest). Child 1's $45,000 of wages belong on the child's own return: they are not part of the parents' AGI, and the child's tax is not added to this output. Under 2026 OBBBA law, the parents get the $32,200 MFJ standard deduction plus the new $2,000 non-itemizer charitable deduction, leaving $65,802 of taxable income. The 10%/12% brackets (10% bracket ends at $24,800) give $7,400.24 of tax, and the $2,200 CTC is fully nonrefundable against it, for $5,200.24. No qualified overtime deduction applies because Head is not flagged as paid hourly. The models went wrong in four main ways: putting the child's wages in the joint AGI or adding the child's separate tax; assuming the TCJA expired; leaving out the $2,000 non-itemizer charitable deduction; and inventing deductions such as mortgage interest or overtime. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_123,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The Child Tax Credit is a nonrefundable credit first; the additional child tax credit is only the residual the taxpayer cannot use against liability, further capped at $1,700 per child in 2026 and at 15% of earnings above $2,500. This household's $145,002 of income less the 2026 MFJ standard deduction leaves roughly $112,800 of taxable income and about $14,000 of federal income tax before refundable credits, which absorbs the entire CTC for the 16-year-old, and the one-child EITC is long gone at joint income above $57,000 — so total federal refundable credits are $0. Every wrong model stopped at the earned-income phase-in and the $400,000 MAGI phase-out and paid out a per-child refundable cap ($1,500, $1,600, $1,700, $2,000, or $2,400) as if the ACTC were a standalone entitlement. One model additionally fabricated a $205 EITC at an income level far past its phase-out."
us,scenario_123,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires both an applicable categorical pathway and satisfaction of that pathway's financial test. The 39-year-old head has no qualifying Medicaid category, and household MAGI is 5.31 times FPL, so the head fails both the categorical and income requirements."
us,scenario_123,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_123,local_income_tax,2,llm_error,state_local_rule,False,,"Pennsylvania residence alone does not establish residence or employment in Philadelphia, and the benchmark's local-income-tax output applies the Philadelphia wage tax only when the relevant locality is specified. Because no Philadelphia locality was listed and unlisted facts must be treated as false, neither the head's nor the child's wages enter a Philadelphia wage-tax calculation, yielding $0."
-us,scenario_123,payroll_tax,35,llm_error,state_local_rule,False,,"Most models got federal FICA right: 6.2% Social Security on $145,000 is $8,990 and 1.45% Medicare is $2,102.50, for $11,092.50. The shared trap is Pennsylvania's mandatory employee unemployment compensation contribution. It is 0.07% of all gross wages with no wage cap, so it adds $101.50 ($70 + $31.50) and brings the total to $11,194. Most models said PA has no employee payroll tax or never raised it. The rest capped the PA contribution at $10,000 of wages, used the wrong rate, computed the right total and then submitted a different number, or made arithmetic errors or added a spurious Additional Medicare Tax."
+us,scenario_123,local_income_tax,3,llm_error,state_local_rule,False,,"The household lists only state PA. No county or city is given, so it is not placed in Philadelphia. The local_income_tax output covers only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. A PA household outside Philadelphia gets none of these, so the value is $0. All three models assumed a Philadelphia residence (or a general PA municipal EIT) that the prompt never states, even though it says to treat unlisted facts as false and not to infer them, and taxed the $145,000 in household wages."
+us,scenario_123,payroll_tax,36,llm_error,state_local_rule,False,,"The household has $145,000 of wages: $100,000 for the head and $45,000 for Child 1. Employee FICA is 7.65% of that, or $8,990 Social Security plus $2,102.50 Medicare. No Additional Medicare Tax applies. Pennsylvania also requires an employee unemployment compensation contribution of 0.07% on all wages with no cap, which adds $101.50 and brings the total to $11,194. Most models stopped at $11,092.50 because they said Pennsylvania has no mandatory employee payroll tax; the rest dropped a figure they had computed, used the wrong UC rate or wage base, invented Additional Medicare Tax, or made arithmetic errors."
us,scenario_123,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires qualification through a specific categorical pathway and its applicable income test. The 39-year-old spouse has no qualifying category and MAGI of 5.31 times FPL, so the spouse fails every available pathway and is not eligible."
us,scenario_123,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_123,state_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"The household is one tax unit: the head and spouse file jointly, and the 16-year-old is their dependent. Pennsylvania's joint return taxes only the spouses' own compensation and interest ($100,000 + $1 + $1 = $100,002) at the flat 3.07% rate, giving $3,070.06. The dependent child's $45,000 of wages is not part of this unit's income. Almost every model added the child's wages to the base ($145,002 × 3.07% = $4,451.56), either by pooling them or by adding a separate return for the child. The few other models invented PA deductions, used the wrong rate or base, or submitted numbers that contradict their own arithmetic."
+us,scenario_123,state_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The prompt puts everyone in one tax unit: the head and spouse file jointly, and the 16-year-old is their dependent. The child's $45,000 of wages is not reported on the parents' return, and no separate child return is computed. Pennsylvania's flat 3.07% tax therefore applies only to the couple's $100,000 of compensation plus $2 of interest, with no standard deduction, exemptions, or itemized deductions: $100,002 × 0.0307 = $3,070.06. Almost every model added the child's $45,000, either by pooling it into the household base or by adding a separate child return, and arrived at about $4,451.56. A few models invented PA deductions or made arithmetic errors. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_123,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv
index 79910701..8a954674 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv
@@ -6,7 +6,7 @@ us,scenario_004,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spo
us,scenario_004,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in 2026, resulting in an is_chip_eligible value of False. CHIP is a program designed to provide health insurance coverage to children and pregnant women in families with incomes above Medicaid limits but below CHIP income thresholds. The head of household, who is 62 years old, does not meet CHIP's age eligibility criteria, as the program does not cover elderly adults. Additionally, the household's income of approximately $52,540 and the head's age place them outside the scope of CHIP's target population, which focuses on minors and pregnant women. Therefore, PolicyEngine correctly classified the head as ineligible for CHIP benefits.",
us,scenario_079,head_medicare_eligible,0.0,1,"PolicyEngine determined that neither adult in this Arizona household is Medicare eligible in 2026, resulting in a value of False for both household members. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old to qualify for the program. Since this benchmark household consists of two working-age adults with a combined household income of approximately $7,560, neither member meets the age threshold for Medicare enrollment. The computation therefore returned False for both individuals, indicating neither qualifies for Medicare benefits in the 2026 tax year.",
us,scenario_118,federal_income_tax_before_refundable_credits,0.0,7,"For this single adult in New York with approximately $2,800 in household income during tax year 2026, PolicyEngine calculated federal income tax before refundable credits as $0. This result derives from the income_tax_before_credits component, which evaluated to $0, indicating that the household's income fell below the threshold at which federal income tax liability is incurred. The other contributing components—net investment income tax, recapture of investment credit, unreported payroll tax, qualified retirement penalty, and income tax capped non-refundable credits—all evaluated to $0, as none of these additional tax mechanisms applied to this household's circumstances. With an annual income of roughly $2,800, the household remained well below the 2026 standard deduction for a single filer, resulting in no federal income tax obligation before any refundable credits are considered.",
-us,scenario_025,state_income_tax_before_refundable_credits,1921.57275390625,77,"PolicyEngine calculated Ohio state income tax before refundable credits of $1,921.57 for this joint household by first determining their Ohio adjusted gross income (AGI) of $94,925.30. From this AGI, PolicyEngine subtracted Ohio personal exemptions totaling $3,800 (two exemptions of $1,900 each for the married couple), resulting in taxable income of $91,125.30. Applying Ohio's tax brackets and rates to this taxable income produced a tax liability of $2,121.57 before credits. Finally, PolicyEngine applied a non-refundable retirement credit of $200, which the household qualified for based on $32,200 in taxable pension income, reducing the final tax before refundable credits to $1,921.57.",
+us,scenario_025,state_income_tax_before_refundable_credits,1916.606201171875,89,"PolicyEngine calculated Ohio income tax before refundable credits of $1,916.61 for this joint return in 2026. Ohio deducts unreimbursed medical care above 7.5% of federal adjusted gross income (R.C. 5747.01(A)(10)). The head paid $6,500 of health insurance premiums and $800 of other medical expenses. Because the household can get an employer-paid plan, the premiums go on line 3 of Ohio's unreimbursed medical care worksheet and the other expenses on line 4, $7,300 in all. Federal adjusted gross income is $94,925.30, so the 7.5% floor is $7,119.40 and the deduction is $180.60, which leaves Ohio adjusted gross income of $94,744.70. Less $3,800 of personal exemptions, taxable nonbusiness income is $90,944.70, taxed at $2,116.61. A $200 pension-based retirement income credit, for the spouse's $32,200 of taxable pension income, lowers the tax to $1,916.61.",
us,scenario_060,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Texas household because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $62,775 in income derives from wage and salary sources rather than self-employment, no self-employment tax liability was triggered. The calculation correctly reflects that self-employment tax applies exclusively to individuals operating as sole proprietors, partners, or independent contractors, none of which apply to this household's income profile.",
us,scenario_025,state_refundable_credits,0.0,46,"PolicyEngine calculated state refundable credits of $0 for this Ohio joint household with approximately $94,925 in income and no children. The computation aggregates refundable tax credits across all 50 states and select localities by summing individual state-level variables. Since the household resides in Ohio (OH = True), PolicyEngine evaluated oh_refundable_credits, which returned $0, as did all other state and locality refundable credit variables (al_refundable_credits through wv_refundable_credits). The final state_refundable_credits value of $0 reflects that Ohio offers no refundable state tax credits applicable to this household's circumstances in tax year 2026.",
us,scenario_040,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not CHIP-eligible (is_chip_eligible = False) for the 2026 tax year in Arizona. The spouse, age 69, fails to meet CHIP's eligibility criteria on the basis of age alone, as CHIP coverage is restricted to children and young adults under age 19 (or up to age 26 in some states with extended programs), and the spouse at 69 years old far exceeds this threshold. Additionally, PolicyEngine confirmed the spouse is not Medicaid-eligible (category: NONE), which would have been an alternative pathway to public coverage. Since the spouse does not satisfy CHIP's age requirement and is ineligible for Medicaid, the engine correctly returned False for CHIP eligibility.",
@@ -65,7 +65,7 @@ us,scenario_005,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employ
us,scenario_042,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Wisconsin household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $58,434 in total income derives from sources other than self-employment (such as wages or salary), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Consequently, PolicyEngine correctly determined that no self-employment tax liability exists for this household in tax year 2026.",
us,scenario_080,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Pennsylvania resident with ~$940 household income in tax year 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with pa_refundable_tax_credits specifically evaluated for this household since the PA state indicator is true. All state-level refundable credit variables, including Pennsylvania's, returned $0, resulting in a total state_refundable_credits value of $0. This outcome reflects that the household does not qualify for any state-administered refundable tax credits based on their income level and household composition.",
us,scenario_100,state_refundable_credits,473.1777038574219,52,"PolicyEngine calculated state refundable credits of $473.18 for this Montana head-of-household with two children and ~$5,915 annual income. Since the household resides in Montana (MT = True), only the Montana state refundable credits component was evaluated, while all other state programs returned zero. The Montana refundable credits derive entirely from the Montana Earned Income Tax Credit (EITC), which computed to $473.18 for the tax unit head. This amount represents 20% of the federal EITC value of $2,365.89, reflecting Montana's standard state EITC rate that provides a percentage of the federal credit to eligible low-income working families.",
-us,scenario_005,state_income_tax_before_refundable_credits,41051.51171875,73,"PolicyEngine calculated California state income tax before refundable credits of $41,051.51 for this joint-filing household with ~$585,777 in income. The calculation began with adjusted gross income of $536,764.50, from which California deductions of $15,338.79 were subtracted (comprising the standard deduction of $11,412 and itemized deductions of $15,338.79, which included $17,280 in real estate taxes subject to limitation). This produced taxable income of $521,425.72, which was then subject to California's tax rate schedule to generate a tax before credits of $41,208.45. Finally, non-refundable credits of $156.93 (California exemptions) were subtracted to arrive at the final state income tax before refundable credits of $41,051.51.",
+us,scenario_005,state_income_tax_before_refundable_credits,40975.25390625,91,"PolicyEngine calculated California income tax before refundable credits of $40,975.25 for this joint filer household. Federal AGI is $543,076.00, and California AGI is $538,928.31 after a $4,147.68 SALT-refund subtraction. California itemized deductions include $17,280.00 of real estate taxes, $2,573.53 of charitable deductions, and $2,986.30 of miscellaneous deductions, totaling $22,839.83 before limitation. After a $2,319.90 overall reduction, itemized deductions are $20,519.93, exceeding the $11,412.00 standard deduction. Subtracting the selected $20,519.93 deduction from California AGI gives taxable income of $518,408.38. California income tax before credits is $41,089.25. Subtracting the $114.00 non-refundable exemption credit gives the reference value of $40,975.25.",
us,scenario_025,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in Ohio for tax year 2026. The primary reason is that the head is 61 years old, which exceeds CHIP's age eligibility threshold that typically covers children and young adults up to age 18 or 19 depending on state rules. Additionally, even if age were not a barrier, the household income of approximately $94,925 would likely exceed Ohio's CHIP income limits for any eligible category. Since the person fails both CHIP's age requirement and income criteria, PolicyEngine correctly returned is_chip_eligible as False.",
us,scenario_118,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this single-adult household in New York in 2026. The household's annual income of approximately $2,800 places it above the income eligibility thresholds for TANF in New York State, which has strict income limits for assistance programs. Additionally, TANF eligibility in New York requires the presence of dependent children in the household, and this benchmark household contains zero children. Since the household fails to meet both the income requirement and the family composition requirement, PolicyEngine correctly determined that no TANF benefit applies.",
us,scenario_060,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. The household head is 70 years old, which exceeds CHIP's age eligibility requirements—the program serves children and pregnant women, not seniors. Additionally, with a household income of approximately $62,775, the person does not meet CHIP's income thresholds for the eligible populations. Since the person fails both CHIP's age criterion and income criterion, they are classified outside any eligible CHIP category, resulting in ineligibility.",
@@ -179,7 +179,7 @@ us,scenario_007,federal_refundable_credits,0.0,6,"PolicyEngine calculated federa
us,scenario_062,head_medicare_eligible,1.0,1,"PolicyEngine determined that the household head is Medicare eligible (value = 1.0, or True) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65. Based on the household composition provided—a single adult in Florida with a household income of approximately $37,642—PolicyEngine's calculation indicates this individual meets the age requirement for Medicare enrollment. The is_medicare_eligible variable therefore returns True, reflecting that this household member qualifies for Medicare benefits under standard federal eligibility rules.",
us,scenario_026,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC benefits, resulting in a value of 0.0 (False). The WIC (Women, Infants, and Children) program eligibility determination for this household member was based on the household's income of approximately $85,209 and composition of 2 adults and 3 children in North Carolina for tax year 2026. Since the spouse did not meet the categorical and income requirements for WIC—which typically limits eligibility to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5 whose household income falls below 185% of the federal poverty line—PolicyEngine returned False for the spouse's WIC eligibility status.",
us,scenario_081,snap,0.0,1,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Massachusetts household in 2026. The household's annual income of approximately $174,088 substantially exceeds the income eligibility threshold for SNAP, which is set at 130% of the federal poverty line for most households. Since this single-adult household's income far surpasses the maximum allowable limit, the household does not qualify for any SNAP benefits in this tax year.",
-us,scenario_043,state_refundable_credits,19.0,54,"PolicyEngine calculated $19 in state refundable credits for this Colorado resident, which came entirely from the Colorado sales tax refund program. The household qualified for this refund because the filer is age 66 and had employment income of $4,072.71, meeting the eligibility criteria for Colorado's sales tax refund. The refund amount of $19 was computed based on the household's modified adjusted gross income of $4,625.40. All other states' refundable credit programs returned zero, as the household resides in Colorado and is therefore only eligible for Colorado's state-level refundable credits.",
+us,scenario_043,state_refundable_credits,0.0,46,"PolicyEngine calculated Colorado state refundable credits of $0.00 for this household in 2026. The household meets the eligibility rules for Colorado's sales tax refund, but C.R.S. 39-22-2003(2) pays that refund for a tax year only if the state had excess revenues in the fiscal year ending in that tax year. The forecasts published before the reference freeze put fiscal year 2025-26 below the Referendum C cap, with no tax year 2026 refund, so PolicyEngine pays none. The household's other Colorado refundable credits are also $0, so its state refundable credits total $0.00.",
us,scenario_100,child1_head_start_eligible,0.0,1,"PolicyEngine determined that neither child in this Montana household is eligible for Head Start in 2026, resulting in a value of False for both children. Head Start eligibility is primarily determined by federal income thresholds, which are typically set at or below 100% of the federal poverty line, though some programs allow up to 130% of poverty. With a household income of approximately $5,915 and two children, this household's income-to-poverty ratio likely exceeded the program's eligibility threshold, disqualifying both children from Head Start enrollment. The resulting array [False, False, False] reflects the ineligibility determination for each household member evaluated against the program's income and other qualifying criteria.",
us,scenario_043,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Colorado single adult household with approximately $4,642 in annual income for tax year 2026. Colorado does not impose a local income tax at the municipal or county level, with income taxation limited to the state level. Therefore, regardless of the household's income amount, the local income tax liability remains $0. This reflects the actual tax code structure in Colorado, where local governments rely on property taxes, sales taxes, and other revenue sources rather than local income taxes.",
us,scenario_100,child1_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this Montana household are eligible for Early Head Start, resulting in an eligibility array of [False, False, False]. Early Head Start is a federal program that serves infants and toddlers from low-income families, typically those under age 3 and with household income at or below 100% of the federal poverty line. Although this household's income of approximately $5,915 is well below the poverty threshold, the children's ages do not meet the program's age requirements for Early Head Start participation. The computation therefore returned False for each child's eligibility status.",
@@ -201,7 +201,7 @@ us,scenario_026,child3_wic_eligible,0.0,1,"PolicyEngine determined that the thir
us,scenario_081,head_wic_eligible,0.0,1,"PolicyEngine determined that the household head is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False for is_wic_eligible. WIC eligibility in Massachusetts requires that household members meet specific criteria, primarily including the presence of pregnant women, postpartum women, breastfeeding women, infants, or children under age 5. This benchmark household consists of a single adult with no children and a household income of approximately $174,088, which means it lacks the required family composition for WIC eligibility. Since the household does not contain any of the qualifying individuals (pregnant women, postpartum women, breastfeeding women, infants, or children under 5), PolicyEngine correctly classified the head as ineligible for WIC benefits.",
us,scenario_100,free_school_meals_eligible,1.0,279,"This Montana household qualifies for free school meals with an annual benefit value of $2,261.92. The household meets eligibility through categorical qualification via SNAP receipt of $8,625.89 annually, which automatically confers free meal status regardless of income level. Additionally, the household's countable income of $11,979.68 (comprising $5,914.72 in employment income and $6,064.96 in TANF benefits) represents 44% of the federal poverty guideline, well below the 130% income threshold for free meals. The two school-age children in the household receive a daily subsidy of $7.15 per child, generating the total annual benefit of $2,261.92 based on standard school year calculations for the contiguous United States.",
us,scenario_043,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this single-adult, zero-child household in Colorado for tax year 2026. TANF is a cash assistance program primarily designed for families with dependent children, and this household does not meet the eligibility criteria as it contains no children. With household income of approximately $4,642 and no qualifying dependents, the household is ineligible for TANF benefits regardless of income level. Therefore, PolicyEngine correctly derived a benefit value of $0.",
-us,scenario_064,federal_income_tax_before_refundable_credits,4439.28955078125,69,"PolicyEngine calculated the federal income tax before refundable credits as $4,439.29 by starting with an IRS gross income of $134,329.59, which included $94,979.60 in employment income (after $2,701.30 in pre-tax 401(k) contributions), $19,350 in partnership/S-corp income, $20,000 in taxable retirement distributions, partially offset by a $16,800 farm loss and $18,235.41 in long-term capital losses. After applying above-the-line deductions of $19,926.22 (including a $19,800 loss deduction and $126.22 in IRA contributions), the adjusted gross income was $114,403.37. The taxable income was further reduced to $67,794.08 through $46,609.29 in deductions, which included the standard deduction of $32,200, a qualified business income deduction of $510, and an overtime income deduction of $13,899.29. Applying 2026 tax rates to this taxable income produced an income tax before credits of $7,639.29, which was then reduced by $3,200 in non-refundable child tax credits to yield the final value of $4,439.29.",
+us,scenario_064,federal_income_tax_before_refundable_credits,4441.455078125,71,"PolicyEngine calculated federal income tax before refundable credits of $4,441.46 for this Wisconsin joint return in 2026. Gross income is $134,329.59: the head's $97,295 of wages less $2,315.40 of pre-tax contributions ($94,979.60), $19,350 of partnership and S corporation income and $20,000 of taxable retirement distributions. Above-the-line deductions total $19,908.19: a $19,800 loss deduction, for the $16,800 farm loss and $3,000 of the $18,235.41 capital loss, and the head's $108.19 traditional IRA contribution. A dependent's $18.03 IRA contribution stays off the filers' return. That leaves adjusted gross income of $114,421.41. Less the $32,200 standard deduction, a $510 qualified business income deduction and a $13,899.29 overtime deduction, taxable income is $67,812.12, taxed at $7,641.46. The child tax credit, $2,200 for the child and $500 for each of two other dependents, takes off $3,200, leaving $4,441.46.",
us,scenario_064,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this Wisconsin household in 2026. The computation evaluated five potential refundable credit components—the Earned Income Tax Credit (EITC), refundable American Opportunity Credit, refundable Child Tax Credit (CTC), Recovery Rebate Credit, and refundable payroll tax credit—and determined that none applied to this household. With a household income of approximately $98,810 and composition of four adults and one child, the household's income level and family structure did not qualify it for any of these refundable credits in the 2026 tax year.",
us,scenario_007,state_refundable_credits,155.0,76,"For this Idaho resident in 2026, PolicyEngine calculated state refundable credits of $155, which derives entirely from the Idaho Grocery Credit. The household qualified for all 12 months of the tax year, with each month meeting the qualifying criteria for the grocery credit program. The base grocery credit amount was determined to be $155, and since the household had no disqualifying factors and met all eligibility requirements throughout the year, this full base amount was awarded as a refundable credit. All other states' refundable credit programs returned zero values, as the household resides in Idaho and does not qualify for credits in other jurisdictions.",
us,scenario_100,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Montana household is not eligible for reduced-price school meals (reduced_price_school_meals_eligible = 0.0) because the household qualifies for the superior free meals tier instead. With a household income of approximately $5,915 and an school_meal_fpg_ratio of 0.44 (meaning income is 44% of the federal poverty guideline), the household falls well below the 130% income threshold for free meals. Additionally, the household meets categorical eligibility requirements, likely through SNAP or TANF participation. Since the household qualifies for free meals via the school_meal_tier classification, the reduced-price tier is not applicable. The annual school_meal_net_subsidy of $2,261.92 reflects the value of the free meals benefit the household receives instead.",
@@ -218,7 +218,7 @@ us,scenario_081,head_medicare_eligible,0.0,1,"PolicyEngine determined that the h
us,scenario_007,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this household in 2026. The head of household is age 56 and does not meet SSI's eligibility criteria, which require the individual to be aged 65 or older, blind, or disabled—none of which apply to this person based on the engine's inputs. Since SSI eligibility is determined on an individual basis rather than at the household level, and no member of this single-adult household qualifies under the aged, blind, or disabled requirement, the household receives no SSI benefit regardless of income level. Although the computation trace shows the household would take up SSI if eligible (takes_up_ssi_if_eligible = True), the absence of a qualifying condition results in zero benefit.",
us,scenario_119,child1_chip_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for CHIP (Children's Health Insurance Program) in Virginia for tax year 2026. The child, who is 14 years old, fails to meet CHIP's eligibility criteria despite the household's income of approximately $55,800. While the household income alone might fall within CHIP's income thresholds in Virginia, the child does not satisfy CHIP's own age or categorical requirements for enrollment. Additionally, the child was determined to be ineligible for Medicaid (with an eligibility status of NONE), which further confirms the lack of public health insurance coverage through either program. As a result, is_chip_eligible is set to False.",
us,scenario_043,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. CHIP eligibility requires meeting both age and income thresholds, and this household fails on the age criterion: at 66 years old, the head exceeds CHIP's maximum age limit, which typically covers children and young adults up to age 18 or 19 depending on state rules. Additionally, with a household income of approximately $4,642, while this income level might otherwise qualify under CHIP's income limits in Colorado, the age disqualification is the binding constraint. The combination of these factors—age ineligibility and the chip_category classification—results in the False determination for CHIP eligibility.",
-us,scenario_064,state_income_tax_before_refundable_credits,4605.9970703125,64,"PolicyEngine calculated Wisconsin state income tax before refundable credits of $4,606 for this joint-filing household. The calculation began with an adjusted gross income (AGI) of $114,403.37, from which PolicyEngine subtracted Wisconsin's standard deduction of $8,759.12 and a personal exemption of $3,500 (based on 5 exemptions in the household) to arrive at taxable income of $102,144.25. This taxable income was then subject to Wisconsin's tax brackets and rates for the 2026 tax year, resulting in the final tax liability of $4,606 before any refundable credits are applied.",
+us,scenario_064,state_income_tax_before_refundable_credits,4599.62158203125,84,"PolicyEngine calculated Wisconsin income tax before refundable credits of $4,599.62 for this joint return in 2026. Wisconsin adjusted gross income equals federal adjusted gross income, $114,421.41: it deducts the $16,800 farm loss, $3,000 of the $18,235.41 capital loss (Wisconsin's limit for a joint return is also $3,000) and the head's $108.19 traditional IRA contribution, but not a dependent's $18.03 contribution, which stays off the filers' return. Less the $8,953.27 sliding-scale standard deduction and $3,500 of exemptions for five people, taxable income is $101,968.14, taxed at $4,599.62 under Wisconsin's 2026 rates, with no nonrefundable credits.",
us,scenario_007,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this single-adult household in Idaho for tax year 2026. With a household income of approximately $54,565, this household exceeds the income eligibility thresholds for TANF assistance in Idaho. TANF is a means-tested program primarily designed for families with children, and this household contains no children, making it ineligible for benefits under the program's eligibility requirements. Therefore, PolicyEngine correctly determined that no TANF benefit applies to this household.",
us,scenario_101,payroll_tax,7879.5,5,"PolicyEngine calculated a payroll tax of $7,879.50 for this single Texas resident with approximately $101,830 in household income. This total comprises two components: employee Social Security tax of $6,386 and employee Medicare tax of $1,493.50. The Social Security tax was computed by applying the 6.2% employee rate to earnings up to the 2026 Social Security wage base limit, while the Medicare tax was calculated by applying the 1.45% employee rate to total wages without an earnings cap. Since the household head is classified as a tax unit head, these employee-side payroll taxes represent the individual's full payroll tax obligation for the year.",
us,scenario_081,free_school_meals_eligible,0.0,17,"PolicyEngine determined that this Massachusetts household is not eligible for free school meals, resulting in a value of 0. Although the state has a universal free school meals program, the household's income of approximately $175,168 places it at 10.98 times the federal poverty guideline (which is $15,960 for a single-person household), far exceeding any income threshold for eligibility. The engine calculated a school_meal_tier of 0, indicating no subsidy qualification, which in turn resulted in a school_meal_net_subsidy of 0 despite the state's universal program being in effect. The household therefore receives no free school meals benefit.",
@@ -234,7 +234,7 @@ us,scenario_026,head_chip_eligible,0.0,1,"PolicyEngine determined that the head
us,scenario_043,free_school_meals_eligible,0.0,282,"PolicyEngine determined that this Colorado household qualifies for free school meals, resulting in a `free_school_meals_eligible` value of 0 (indicating the household receives free meals rather than a reduced or paid tier). The household's income of approximately $4,648 represents 29% of the federal poverty guideline (`school_meal_fpg_ratio` of 0.29), well below the 130% threshold for free meals. Additionally, the household meets categorical eligibility through SNAP participation, receiving a monthly allotment of $304.68. Colorado's universal free school meals program further ensures eligibility. Although the household would qualify for a daily subsidy of $7.15 under standard federal reimbursement rules, the `school_meal_net_subsidy` is calculated as 0 because the state's universal program provides free meals regardless of the federal subsidy mechanism.",
us,scenario_101,state_income_tax_before_refundable_credits,0.0,46,"PolicyEngine calculated a state income tax before refundable credits of $0.00 for this Texas household with approximately $101,830 in income. The computation aggregates income tax liabilities across all 50 states plus Washington D.C. and New York City, with each individual state and local jurisdiction contributing $0.00 to the total. Texas, the household's state of residence, imposes no state income tax, which is the primary driver of the zero result. All other states and localities similarly returned zero values, confirming that no state-level income tax liability was incurred for this benchmark household in tax year 2026.",
us,scenario_119,child1_medicare_eligible,0.0,1,"PolicyEngine determined that child1_medicare_eligible equals False for the benchmark household in Virginia for tax year 2026. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and the household composition shows 2 children whose ages fall well below this threshold. The computation trace shows is_medicare_eligible returned [False, False, False] across all three household members (the head of household and two children), confirming that none qualify for Medicare based on the age requirement. Since the children in this household do not meet the minimum age criterion of 65, child1_medicare_eligible correctly evaluates to False.",
-us,scenario_082,federal_income_tax_before_refundable_credits,9563.052734375,84,"PolicyEngine calculated federal income tax before refundable credits of $9,563.05 for this New York head-of-household filer with one child and ~$90,132 household income. The calculation began with gross income of $120,593.62, which included $100,013.62 in employment income, $20,480 in dividend income, and $100 in taxable interest income, partially offset by $30,710.82 in net capital losses. After applying above-the-line deductions of $3,008.48 and the standard deduction of $24,150, the filer's taxable income was $92,735.15, which generated $10,349.45 in tax under the ordinary income rates plus an additional $2,013.60 in capital gains tax on $13,424 of qualified dividends and long-term capital gains, totaling $12,363.05 before credits. Non-refundable credits of $2,800—comprising $600 in child and dependent care credit and $2,200 in child tax credit—reduced this to the final amount of $9,563.05.",
+us,scenario_082,federal_income_tax_before_refundable_credits,9579.767578125,129,"PolicyEngine calculated federal income tax before refundable credits of $9,579.77 for this head-of-household filer with one child in 2026. IRS gross income is $120,661.12, comprising $100,013.62 of IRS employment income after $181.37 of pre-tax contributions, $20,480.00 of dividends, $100.00 of taxable interest, and $67.50 of SALT refund income. The household also has a $30,710.82 net capital loss, of which a $3,000.00 loss deduction reduces gross income to AGI of $117,661.12. The $24,150.00 standard deduction plus $700.00 charitable deduction for non-itemizers leaves taxable income of $92,811.12. The calculation excludes $13,424.00 of qualified dividends from main-rate income. Main-rate tax of $10,366.17 plus capital gains tax of $2,013.60 gives $12,379.77 before credits. Subtracting $2,800.00 of non-refundable credits, comprising $600.00 of child and dependent care credit and $2,200.00 of child tax credit, gives $9,579.77.",
us,scenario_064,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Wisconsin household in 2026. The household, consisting of 4 adults and 1 child with a combined income of approximately $98,810, was determined to be ineligible for SNAP benefits based on income thresholds. Although the model indicates that the household would take up SNAP if eligible (takes_up_snap_if_eligible = True), the household's income exceeds the gross income limit for a household of this size in Wisconsin, resulting in zero benefits. The repeated entries in the computation trace reflect PolicyEngine's evaluation across multiple household members, all confirming the same ineligibility determination.",
us,scenario_082,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this head-of-household filer with one child and approximately $90,132 in household income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. This outcome reflects that the household's income level and composition did not qualify for any of the refundable credits available under 2026 tax law.",
us,scenario_026,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (is_chip_eligible = False) in this North Carolina household for tax year 2026. The spouse, age 45, exceeds CHIP's maximum age limit, which typically covers children and young adults up to age 18 or 19 depending on the state program. Additionally, the household income of approximately $85,209 places the family above North Carolina's CHIP income thresholds for adults. Since the spouse fails both CHIP's age requirement and the program's income eligibility criteria, PolicyEngine correctly classified them as ineligible for CHIP benefits.",
@@ -266,8 +266,8 @@ us,scenario_007,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin
us,scenario_101,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Texas household in 2026. The household, consisting of a single adult with no children and an annual income of approximately $101,830, does not qualify for TANF benefits because the program is designed to provide cash assistance to families with dependent children. Since this household has zero children, it falls outside the eligibility criteria for TANF, resulting in no benefit amount regardless of income level.",
us,scenario_044,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Kansas household in 2026 because neither spouse reported self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this joint household's approximately $56,806 in total income derives from wages or other non-self-employment sources, there is no self-employment tax liability. The calculation correctly reflects that self-employment tax applies exclusively to individuals with Schedule C net profit or other qualifying self-employment earnings, which this household does not have.",
us,scenario_119,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this Virginia household qualifies for Early Head Start in 2026, resulting in an eligibility array of [False, False, False]. Early Head Start is a federal program specifically designed for children under age 3, and this household's children do not meet the age requirements for the program. With a household income of approximately $55,800 and head-of-household filing status, the family's income level would not have been a disqualifying factor, as Early Head Start serves low-income families and this income falls within typical eligibility thresholds. The determination reflects that the children in this benchmark household simply fall outside the target age range for this particular early childhood intervention program.",
-us,scenario_082,state_refundable_credits,667.0,102,"PolicyEngine calculated New York refundable credits of $667 for this head of household with one child aged 1. The Empire State child credit is $307: the $1,000 credit for a child aged three or younger, reduced by $16.50 for each whole $1,000 of federal adjusted gross income ($117,652.65) above the $75,000 threshold, 42 steps or $693. New York's child and dependent care credit is $360: 60% of the federal credit, which is 20% of the $3,000 of care expenses the federal credit counts for one child.",
-us,scenario_082,state_income_tax_before_refundable_credits,5598.55224609375,77,"For this New York head-of-household filer with approximately $90,132 in household income, PolicyEngine calculated a state income tax before refundable credits of $5,598.55. The calculation began with an adjusted gross income (AGI) of $117,585.15, from which New York deductions of $11,200 and exemptions of $1,000 were subtracted to arrive at a taxable income of $105,385.15. The main New York income tax on this taxable income was $5,442.17, to which a supplemental tax of $156.38 was added, yielding the final total of $5,598.55. The deductions included both itemized deductions of $11,169.24 (comprising $11,057.16 in SALT deduction and $112.07 in charitable deduction) and the standard deduction of $11,200, with the standard deduction being applied as the larger amount.",
+us,scenario_082,state_refundable_credits,1187.5478515625,95,"PolicyEngine calculated New York state refundable credits of $1,187.55 for this head-of-household return in 2026: a $307 Empire State child credit and an $880.55 child and dependent care credit. The child credit is its $1,000 base less a $693 phase-out at federal adjusted gross income of $117,661.12. The care credit follows Tax Law 606(c-2), which applies from 2026: $3,000 of the household's $9,600 of child care expenses qualify for its one child, and the applicable percentage at New York adjusted gross income of $117,593.62, about 29.352%, gives $880.55.",
+us,scenario_082,state_income_tax_before_refundable_credits,5599.1435546875,97,"PolicyEngine calculated New York income tax before refundable credits of $5,599.14 for this head-of-household filer with one child dependent in 2026. IRS gross income of $120,661.12 comprises $100,013.62 of IRS employment income, $20,480.00 of dividends, $100.00 of taxable interest, and $67.50 of SALT refund income. The household has a $30,710.82 capital loss, of which a $3,000.00 above-the-line loss deduction applies, leaving federal AGI of $117,661.12. Subtracting $67.50 of SALT refund income gives New York AGI of $117,593.62. The selected $11,200.00 standard deduction exceeds itemized deductions of $9,328.22. Subtracting the standard deduction and $1,000.00 dependent exemption gives taxable income of $105,393.62. Main New York tax of $5,442.63 plus supplemental tax of $156.51 gives the final $5,599.14.",
us,scenario_026,spouse_medicare_eligible,0.0,1,"PolicyEngine determined that the spouse is not Medicare eligible in this 2026 household, resulting in a value of False (0.0). Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and the spouse in this North Carolina household does not meet this age threshold. The household composition of two adults and three children with a combined income of approximately $85,209 does not trigger any alternative pathways to Medicare eligibility, such as end-stage renal disease or ALS status. Therefore, PolicyEngine correctly assigned False to the spouse_medicare_eligible variable, indicating no Medicare coverage eligibility for the spouse in tax year 2026.",
us,scenario_026,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible in 2026, returning a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, requiring individuals to be at least 65 years old, along with citizenship or permanent residency status and sufficient work history under Social Security. Based on the household composition provided—a joint filer household with 2 adults and 3 children in North Carolina with a combined income of approximately $85,209—PolicyEngine's calculation indicates that the head of household does not meet the age threshold for Medicare eligibility. The array output [False, False, False, False, False] reflects the eligibility status for all five household members, confirming that none currently qualify for Medicare benefits.",
us,scenario_008,federal_income_tax_before_refundable_credits,0.0,63,"PolicyEngine calculated federal income tax before refundable credits as $0 for this New Jersey joint filer household with 6 children and ~$30,915 in household income. The household's income_tax_before_credits was $0, meaning no federal income tax liability was owed before applying any credits. Although the household had non-refundable credits totaling $7,297.39—primarily composed of $7,297.39 in non-refundable child tax credit (CTC)—these credits exceeded the tax liability, resulting in income_tax_capped_non_refundable_credits of $0. The non-refundable CTC itself was derived from a total CTC of $11,500 (based on 6 qualifying children at $2,200 each, plus one adult dependent at $500), with $4,202.61 of this amount classified as refundable CTC based on the household's earned income of $30,517.41 and resulting CTC phase-in amount. Since the household owed no tax before credits, the final value of income_tax_before_refundable_credits remained $0.",
@@ -281,7 +281,7 @@ us,scenario_008,federal_refundable_credits,12433.611328125,6,"PolicyEngine calcu
us,scenario_026,child1_medicare_eligible,0.0,1,"PolicyEngine determined that child1 is not Medicare eligible (value: False) for the 2026 tax year. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, with limited exceptions for younger individuals with end-stage renal disease or ALS. Since child1 is a dependent child in this North Carolina household, they do not meet the age threshold or qualifying disability criteria for Medicare coverage. The household's income level of approximately $85,209 does not affect Medicare eligibility, which is age-based rather than means-tested. Therefore, PolicyEngine correctly assigned a value of False (0.0) to indicate that child1 is not Medicare eligible.",
us,scenario_044,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Kansas joint household with ~$56,806 income in 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with the household's state of residence (Kansas) determining which state-specific credit applies. Since ks_refundable_credits evaluated to $0, and all other state refundable credit variables (al_refundable_credits, ar_refundable_credits, ca_refundable_credits, and so on through wv_refundable_credits) also returned $0, the sum of all state-level refundable credits equals $0. This result indicates that either Kansas offers no refundable tax credits for this household's income level and filing status, or the household does not qualify for any available refundable credits under 2026 tax law.",
us,scenario_082,snap,0.0,25,"For this New York head-of-household with one child and household income of approximately $90,132 in 2026, PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0. Although the household's take-up behavior was modeled as willing to claim SNAP if eligible (takes_up_snap_if_eligible = True), the household's income level exceeded the eligibility threshold for SNAP benefits. At roughly $90,132 in annual income, this household surpasses the gross income limit for a household of two people in New York, which is why no benefit amount was determined. The repeated computation trace showing consistent $0 values across multiple calculation nodes reflects that the ineligibility determination was consistent throughout PolicyEngine's evaluation of the household's SNAP entitlement.",
-us,scenario_008,payroll_tax,2162.760009765625,18,"PolicyEngine calculated the household's payroll tax at $2,162.76 based on employment income of $26,800 earned by one spouse (the tax unit head). This total comprises three components: federal employee Social Security tax of $1,661.60, federal employee Medicare tax of $388.60, and New Jersey state payroll taxes of $112.56. The New Jersey state portion consists of a temporary disability insurance contribution of $50.92 and a family leave insurance contribution of $61.64, both calculated on the same $26,800 taxable wage base. The other household members (the second adult and six children) generated no payroll tax liability, as indicated by the per-entity breakdown showing all contributions attributable to entity 1 only.",
+us,scenario_008,payroll_tax,2276.659912109375,32,"PolicyEngine calculated payroll tax of $2,276.66 by summing employee payroll taxes. One of the household's eight people has $26,800.00 of employment income, and the other seven have none. Employee Social Security tax is $1,661.60, equal to 6.2% of $26,800.00, and employee Medicare tax is $388.60, equal to 1.45% of that wage amount. Together they total $2,050.20. New Jersey employee state payroll tax adds $226.46: $50.92 for temporary disability insurance, $61.64 for family leave insurance, $102.51 for unemployment insurance, and $11.39 for the workforce fund. Each contribution’s taxable-wage input is $26,800.00. Adding these four contributions to Social Security and Medicare taxes gives the reference payroll tax of $2,276.66.",
us,scenario_119,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Virginia household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $55,800 translates to a school_meal_fpg_ratio of 2.04, meaning their income is about 204% of the federal poverty guideline. Since Virginia does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they must meet the income threshold to access reduced-price meals. The federal reduced-price threshold is set at 185% of the poverty guideline, and because this household's income exceeds that limit at 204%, they are classified in the school_meal_tier of PAID and receive no subsidy (school_meal_net_subsidy = 0), requiring them to pay the full price for school meals.",
us,scenario_026,child2_medicare_eligible,0.0,1,"PolicyEngine determined that child 2 is not Medicare eligible in this household, returning a value of False. Medicare eligibility in the United States is primarily determined by age and work history; individuals must generally be 65 years or older, or qualify through disability or end-stage renal disease. Since child 2 in this North Carolina household is a dependent child (part of a 3-child household with two adults), they do not meet the age threshold of 65 and have no work history qualifying them for disability-based Medicare coverage. Therefore, PolicyEngine correctly computed is_medicare_eligible as False for child 2 in the 2026 tax year.",
us,scenario_082,ssi,0.0,25,"PolicyEngine calculated Supplemental Security Income (SSI) of $0.00 for this household because neither household member meets SSI's eligibility criteria. The head of household, age 23, does not satisfy the aged (65+), blind, or disabled requirement, and therefore is_ssi_aged_blind_disabled is False with an SSI benefit of $0.00. Similarly, the child, age 1, does not meet any of these eligibility thresholds, also resulting in is_ssi_aged_blind_disabled=False and $0.00 in SSI. Since SSI eligibility is determined on an individual basis rather than at the household level, and no member qualifies under the program's categorical requirements, the household receives no SSI benefit regardless of income level or take-up behavior.",
@@ -311,7 +311,7 @@ us,scenario_008,local_income_tax,0.0,1,"PolicyEngine calculated a local income t
us,scenario_026,child3_head_start_eligible,0.0,1,"PolicyEngine determined that the third child in this North Carolina household is not eligible for Head Start (value: False). Head Start eligibility is primarily determined by federal income thresholds, which are typically set at or below 100% of the federal poverty line or 130% of the federal poverty line depending on program rules. With a household income of approximately $85,209 and three children, this household's income-to-poverty ratio exceeds the eligibility threshold for the 2026 tax year. Since the household income substantially surpasses the income limit required to qualify for Head Start services, PolicyEngine correctly classified the third child as ineligible, resulting in the False value for is_head_start_eligible.",
us,scenario_044,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this Kansas household is eligible for the Women, Infants, and Children (WIC) program, resulting in an `is_wic_eligible` value of [False, False]. WIC eligibility requires the presence of children under age 5 in the household, and this benchmark household contains 0 children. Since the fundamental eligibility criterion—having at least one infant or young child—is not met, both household members are ineligible regardless of their household income of approximately $56,806 or other factors. The program's child-dependent structure means that childless households cannot qualify for WIC benefits under any circumstances.",
us,scenario_102,federal_income_tax_before_refundable_credits,6104.09619140625,40,"PolicyEngine calculated the federal income tax before refundable credits as $6,104.10 for this North Carolina joint filer household with approximately $87,201 in income. The calculation began with gross income of $87,201, comprised primarily of $87,200 in employment income split between the two spouses ($41,200 and $46,000) plus $1 in self-employment income. From this gross income, PolicyEngine applied taxable income deductions totaling $32,200.20, which consisted of the standard deduction of $32,200 plus a qualified business income deduction of $0.20, resulting in taxable income of $55,000.80. The income tax before refundable credits of $6,104.10 was then derived by applying the 2026 federal tax rate schedule to this taxable income, with no adjustments from net investment income tax, credits, or other modifications.",
-us,scenario_120,state_income_tax_before_refundable_credits,11917.18359375,75,"PolicyEngine calculated Connecticut state income tax before refundable credits of $11,917.18 for this single filer with ~$244,169 household income. The calculation began with an adjusted gross income (AGI) of $232,635.12, from which Connecticut-specific AGI subtractions of $25,140 were applied—primarily a Social Security benefit adjustment of $25,140 based on $41,900 in Social Security benefits and combined income calculations. This produced a Connecticut AGI of $207,495.12, which was then reduced to a taxable income of $207,495.12 after accounting for phase-out add-backs of $250 and recapture adjustments totaling $430 (comprising $250 in low-tax recapture and $180 in middle-tax recapture). The resulting tax before any personal credits and alternative minimum tax was $11,917.18, which remained unchanged after applying the AMT and personal credit calculations, as no credits reduced the liability.",
+us,scenario_120,state_income_tax_before_refundable_credits,11919.0576171875,87,"PolicyEngine calculated Connecticut income tax before refundable credits of $11,919.06 for this household in 2026. IRS gross income is $236,860.80. Positive income inputs include $164,979.56 of IRS employment income, $35,615.00 of taxable Social Security, $30,216.00 of taxable pension income, $2,300.00 of taxable interest, $2,000.00 of rental income, and $1,750.24 of SALT refund income. A $2,446.59 above-the-line loss deduction leaves federal AGI of $234,414.20. Connecticut AGI subtractions total $26,890.23, comprising a $25,140.00 Social Security benefit adjustment and the SALT refund subtraction. Connecticut AGI and taxable income are $207,523.97. The tax calculation also uses a $250.00 phase-out add-back and $430.00 of recapture adjustments, comprising $250.00 of low-tax recapture and $180.00 of middle-tax recapture. The resulting tax before refundable credits is $11,919.06.",
us,scenario_026,child1_early_head_start_eligible,0.0,1,"PolicyEngine determined that none of the three children in this North Carolina household are eligible for Early Head Start in 2026, resulting in a value of False for all household members. Early Head Start eligibility is typically restricted to children under age 3, and this household's children do not meet the age requirements for the program. With a household income of approximately $85,209, the family's income level would not present a barrier to eligibility if age requirements were met, as Early Head Start serves low-income families and this income falls within the range of many state programs. The False determination across all five household members reflects that either the children exceed the age threshold or the household composition does not include infants and toddlers in the target age range for this early childhood education program.",
us,scenario_044,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of 0.0 (False). The WIC program is specifically designed to serve pregnant women, postpartum women, breastfeeding women, infants, and children under age 5 who meet income and nutritional risk criteria. Since this benchmark household consists of 2 adults with no children and a household income of approximately $56,806, the spouse does not fall into any of the eligible demographic categories for WIC participation. Therefore, PolicyEngine correctly identified ineligibility based on the household composition and age requirements of the program.",
us,scenario_008,snap,15108.0,541,"PolicyEngine computes an annual 2026 SNAP benefit of $15,108 for this eight-person New Jersey household, made up of $1,259 in each of the twelve months. Monthly gross income is $2,576.25: $2,566.67 in earned income (the spouse's $2,233.33 in wages plus the head's self-employment income) and $9.58 in interest. Subtracting a $299 standard deduction and a $513.33 earned income deduction, $812.33 in total, leaves $1,763.92, which rounds to net income of $1,764. The household passes the gross and net income tests and is categorically eligible through TANF non-cash eligibility in New Jersey; gross income is 57% of the poverty guideline from January through September and 55% from October through December. Net income sets an expected contribution of $530, and the $1,789 maximum allotment for eight people minus $530 gives $1,259 a month.",
@@ -563,7 +563,7 @@ us,scenario_084,head_medicare_eligible,0.0,1,"PolicyEngine determined that the h
us,scenario_067,ssi,0.0,25,"For this Indiana household in 2026, PolicyEngine calculated Supplemental Security Income (SSI) of $0.00 across all three household members. SSI eligibility requires that an individual meet at least one of three criteria: being aged 65 or older, blind, or disabled. In this case, none of the household members satisfied these requirements—the head of household is age 60, the spouse is age 58, and the dependent is age 23, and none are flagged as blind or disabled in the engine's inputs. Since PolicyEngine evaluates SSI eligibility on an individual basis (with spousal income deeming rules applied where applicable), each person's ineligibility resulted in $0.00 in SSI benefits. Although all three members indicated willingness to take up SSI if eligible (takes_up_ssi_if_eligible = True for each), the absence of a qualifying aged, blind, or disabled status meant no benefits were payable regardless of household income level.",
us,scenario_084,free_school_meals_eligible,0.0,15,"PolicyEngine determined that this North Carolina household is not eligible for free school meals, resulting in a value of 0. The household's school_meal_countable_income of $24,833—comprising $24,828 in social security and $5 in interest income—yields a school_meal_fpg_ratio of 1.56, meaning the household's income is 156% of the federal poverty guideline (spm_unit_fpg of $15,960). This ratio places the household in the REDUCED tier for school meal pricing rather than the free tier, as the household exceeds the income threshold for free meals. Since the state of North Carolina does not have universal free school meals and the household does not meet categorical eligibility through programs like SNAP or TANF, the household qualifies only for reduced-price meals with a net subsidy of $0 for free meals.",
us,scenario_107,state_income_tax_before_refundable_credits,0.0,47,"PolicyEngine calculated the state income tax before refundable credits as $0 for this Ohio resident with approximately $49,500 in household income. The computation aggregates income tax liabilities across all 50 states and the District of Columbia, with only the Ohio-specific variable `oh_income_tax_before_refundable_credits` being relevant given the household's Ohio residency (indicated by `OH = True`). Since `oh_income_tax_before_refundable_credits` equals $0, the overall `state_income_tax_before_refundable_credits` also equals $0. This result reflects Ohio's tax structure, where the household's income level and filing status (single, no dependents) results in no state income tax liability before any refundable credits are applied.",
-us,scenario_067,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0.00 for this Indiana household in 2026. Indiana does not impose a local income tax at the state level, and the household's location within Indiana does not have a municipal income tax that would apply to their wages and other income sources. Therefore, despite the household's joint income of approximately $98,102, no local income tax liability was generated under the applicable tax rules for the 2026 tax year.",
+us,scenario_067,local_income_tax,1348.83203125,28,"PolicyEngine computed local income tax of $1,348.83 for this Indiana household, using Adams County. Federal gross income and AGI are both $87,302.00, comprising $82,360.00 of employment income and $4,942.00 of taxable pension income. Indiana applies $3,000.00 of exemptions for a tax unit of three people, leaving Indiana AGI of $84,302.00. County tax of $1,348.83 corresponds to 1.6% of that amount: $84,302.00 × 1.6% = $1,348.832, which rounds to $1,348.83. No other local income tax applies, so the Indiana county tax is the full local income tax of $1,348.83.",
us,scenario_067,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Indiana household in 2026. The household consists of 3 adults with no children and a combined income of approximately $98,102. TANF is a needs-based program primarily designed to assist families with dependent children, and since this household has no children, it does not meet the eligibility requirements for this benefit. Consequently, PolicyEngine assigned a benefit value of $0, reflecting the household's ineligibility for TANF assistance.",
us,scenario_008,child6_head_start_eligible,0.0,1,"PolicyEngine determined that none of the six children in this New Jersey household are eligible for Head Start, resulting in a value of False for all eight household members (the two adults plus six children). Head Start eligibility is primarily determined by federal income thresholds, which are typically set at or below 100% of the federal poverty line, though some programs allow up to 130% of poverty. With a household income of approximately $30,915 and eight total members, this household's income-to-poverty ratio exceeds the standard eligibility cutoff for the 2026 tax year. Consequently, PolicyEngine marked all six children as ineligible (False) for Head Start enrollment.",
us,scenario_084,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. Although the household's income-to-poverty ratio of 1.56 places it in the REDUCED school meal tier (between 130% and 185% of the federal poverty guideline, which qualifies for reduced-price meals under federal guidelines), the household has zero children and therefore no school-age dependents who could benefit from the program. Since the school_meal_net_subsidy is 0 due to the absence of eligible children in the household, the reduced_price_school_meals benefit amount is 0.",
@@ -573,7 +573,7 @@ us,scenario_107,state_refundable_credits,0.0,46,"PolicyEngine calculated state_r
us,scenario_107,local_income_tax,0.0,1,"For a single adult in Ohio with a household income of approximately $49,500 in tax year 2026, PolicyEngine calculated a local income tax liability of $0. This result reflects that the household either resides in a jurisdiction without a local income tax, or falls below any applicable local tax thresholds in their area. Ohio's local income tax structure varies significantly by municipality, with many jurisdictions imposing no local income tax at all, which explains the zero valuation for this benchmark household.",
us,scenario_008,child1_early_head_start_eligible,0.0,1,"PolicyEngine determined that child 1 is not eligible for Early Head Start (value: False). Early Head Start eligibility is primarily determined by age and income thresholds. Child 1 does not meet the age requirement for the program, which serves infants and toddlers under age 3. The household's income of approximately $30,915 is below the typical Early Head Start income eligibility threshold, but this favorable income status cannot override the age restriction. Only the eighth household member (likely the youngest child) was marked as eligible (True), indicating that child meets both the age and income requirements for Early Head Start enrollment.",
us,scenario_067,head_wic_eligible,0.0,1,"PolicyEngine determined that none of the three household members are eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False, False]`. WIC eligibility requires applicants to meet specific categorical requirements—typically being pregnant, postpartum, breastfeeding, or a child under age 5—in addition to income limits set at 185% of the federal poverty line. Since this benchmark household consists of 3 adults with no children and a household income of approximately $98,102, the members do not satisfy the categorical eligibility criteria that WIC mandates. Therefore, PolicyEngine correctly computed that all household members have zero WIC eligibility status.",
-us,scenario_085,federal_income_tax_before_refundable_credits,2322.760009765625,28,"PolicyEngine calculated federal income tax before refundable credits of $2,322.76 for this Pennsylvania single filer by first determining taxable income of $21,423. The household's adjusted gross income of $45,573 was reduced by above-the-line deductions of $0.40 and taxable income deductions of $24,150 (comprising the standard deduction of $18,150 plus an additional senior deduction of $6,000), resulting in the taxable income figure. This taxable income was then subject to the federal income tax main rates, which generated a tax liability of $2,322.76. The calculation also evaluated net investment income tax on the $1,675 in taxable interest income, but this resulted in no additional tax. No other adjustments, credits, or penalties applied, so the final income tax before refundable credits remained $2,322.76.",
+us,scenario_085,federal_income_tax_before_refundable_credits,2322.8486328125,28,"PolicyEngine calculated federal income tax before refundable credits of $2,322.85 for this household in 2026. AGI of $45,573.74 comprises $1,675.00 of taxable interest, $20,720.00 of taxable private pension income, $14,050.74 of taxable Social Security, and $9,128.00 of taxable retirement distributions. Selected taxable-income deductions total $24,150.00: the $18,150.00 standard deduction plus a $6,000.00 additional senior deduction. This exceeds the $10,115.88 deduction total available if itemizing. Subtracting $24,150.00 from AGI leaves taxable income of $21,423.74, producing $2,322.85 of main-rate tax and tax before credits. Net investment income tax is zero, with modified AGI of $45,573.74 and net investment income of $1,675.00 as inputs. Other listed additional taxes and capped non-refundable credits are zero, leaving the final tax of $2,322.85.",
us,scenario_008,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that child 2 is not eligible for Early Head Start, returning a value of False (0.0). Early Head Start eligibility in the United States is primarily determined by age and income thresholds, with the program serving infants and toddlers under age 3 from low-income families. Based on the household composition showing 6 children with a household income of approximately $30,915, PolicyEngine evaluated child 2 against these federal eligibility criteria. The False result indicates that child 2 does not meet the age requirement (likely being 3 years or older) or that the household's income, while modest, may exceed the specific income threshold for Early Head Start eligibility in New Jersey, or that child 2 falls outside the target age range for this particular early childhood program.",
us,scenario_085,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household. The computation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (refundable_ctc), the recovery rebate credit, and the refundable payroll tax credit. Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. For a single adult with no children and household income of approximately $58,949, this household does not qualify for any of these refundable credits under 2026 tax law.",
us,scenario_122,free_school_meals_eligible,0.0,19,"PolicyEngine determined that this Minnesota household is not eligible for free school meals, resulting in a value of 0.0. Although Minnesota has a universal free school meals program, this household does not qualify because it has no children—the free school meals program only applies to school-age children. The household's school_meal_tier was calculated as 0 (not eligible), based on an income-to-poverty ratio (school_meal_fpg_ratio) of 4.95, which reflects countable income of $79,038.35 against the federal poverty guideline of $15,960 for a single-person household. Since there are no children in the household to receive meals, the school_meal_net_subsidy is 0, and therefore the free_school_meals benefit value is 0.0.",
@@ -772,7 +772,7 @@ us,scenario_109,state_refundable_credits,0.0,45,"PolicyEngine calculated state_r
us,scenario_086,child1_chip_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for CHIP in Georgia for tax year 2026. The child fails to meet CHIP's eligibility criteria on multiple grounds: the child is not Medicaid-eligible (with engine category NONE), and additionally does not satisfy CHIP's own independent eligibility requirements regarding either age or income thresholds. Since the child must satisfy both the prerequisite Medicaid ineligibility pathway and CHIP's specific criteria, and fails on both fronts, the is_chip_eligible variable correctly evaluates to False.",
us,scenario_012,state_income_tax_before_refundable_credits,0.0,48,"PolicyEngine calculated the state income tax before refundable credits for this Mississippi household as $0 by aggregating the income tax liabilities across all 50 states and DC. While Mississippi's individual income tax calculation (ms_income_tax_before_credits_indiv) produced $118 in tax liability for the primary earner, the unit-level Mississippi income tax before credits (ms_income_tax_before_credits_unit) was reduced to $0, likely due to the application of standard deductions or other Mississippi-specific exemptions that eliminated the tax obligation at the household level. All other states' income tax variables returned $0, reflecting either no income tax liability in those jurisdictions or the household's lack of connection to them. The final state income tax before refundable credits of $0 represents the sum of all these state-level calculations.",
us,scenario_029,free_school_meals_eligible,0.0,15,"PolicyEngine determined that this Oklahoma household is not eligible for free school meals, resulting in a value of 0. The household's school_meal_countable_income of $50,312—comprising $312 in interest income and $50,000 in financial assistance—yields a school_meal_fpg_ratio of 3.15, meaning the household's income is 315% of the federal poverty guideline (spm_unit_fpg of $15,960). Since this ratio exceeds the 185% threshold for reduced-price meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is classified in school_meal_tier 2 (PAID). Oklahoma does not have universal free school meals, so the household must pay the full price for school meals with no subsidy.",
-us,scenario_049,federal_income_tax_before_refundable_credits,30543.908203125,48,"PolicyEngine calculated federal income tax before refundable credits of $30,543.91 for this New Hampshire joint filer by first determining taxable income of $188,883.95, which started from adjusted gross income of $246,663.72 (comprising $237,564 in employment income after $15,436 in pre-tax 401(k) contributions, plus $3,501 in taxable interest and $6,320 in dividend income). The taxable income was reduced by $57,779.77 in deductions, primarily the standard deduction of $32,200 and an overtime income deduction of $25,000. The income tax before credits was then computed as $29,612.71 using the main tax rate schedule on this taxable income, plus an additional $931.20 in capital gains tax on the $6,208 of qualified dividend income taxed at preferential rates. No net investment income tax, recapture taxes, or other adjustments applied, resulting in the final tax liability of $30,543.91.",
+us,scenario_049,federal_income_tax_before_refundable_credits,30702.58984375,62,"PolicyEngine calculated federal income tax before refundable credits of $30,702.59 for this joint return in 2026. Employment income of $253,000.00 less $15,436.00 of pre-tax contributions gives $237,564.00 of IRS employment income. Adding $3,501.00 of taxable interest and $6,320.00 of dividends gives IRS gross income and AGI of $247,385.00. Taxable-income deductions are $57,779.77, including the $32,200.00 standard deduction, $25,000.00 overtime deduction, and $579.76 charitable deduction for non-itemizers. Taxable income is $189,605.23. Excluding $6,208.00 of qualified dividends leaves $183,397.23 for main rates, producing $29,771.39 of tax. Capital gains tax adds $931.20, giving $30,702.59 before credits. Net investment income tax is zero, with modified AGI of $247,385.00 and net investment income of $9,821.00. Other listed additions and capped non-refundable credits are zero.",
us,scenario_070,payroll_tax,2907.0,5,"PolicyEngine calculated a payroll tax of $2,907 for this single Illinois resident with approximately $38,100 in household income. This total comprises two components: employee Social Security tax of $2,356 and employee Medicare tax of $551. The Social Security tax was computed by applying the standard 6.2% employee contribution rate to the applicable wage base, while the Medicare tax was calculated using the standard 1.45% employee rate on wages. Since this household member is identified as a tax unit head, they are subject to the full employee-side payroll tax obligations without any special exemptions or reductions.",
us,scenario_029,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Oklahoma household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $312 places it at 3.15 times the federal poverty guideline, which exceeds the 185% income threshold required for reduced-price meal eligibility. Since the household does not qualify for categorical eligibility through programs like SNAP or TANF, and Oklahoma does not have universal free school meals, the household is assigned to the PAID school_meal_tier. With no subsidy applied (school_meal_net_subsidy = 0), the household must pay the full price for school meals.",
us,scenario_109,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Florida household in 2026. Florida does not impose a state income tax, and the local_income_tax variable in PolicyEngine's US tax module reflects taxes levied by local jurisdictions within states that permit such taxation. Since Florida has no state income tax framework and does not authorize local income taxes on wages and salaries, there is no local income tax liability for this household regardless of their $30,000 household income or family composition. The $0 value represents the absence of any applicable local income tax obligation under Florida law for the 2026 tax year.",
@@ -953,7 +953,7 @@ us,scenario_051,head_medicare_eligible,0.0,1,"PolicyEngine determined that the h
us,scenario_031,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this California household is not eligible for reduced-price school meals (reduced_price_school_meals = 0.0) because the household qualifies for a better benefit tier. With an income of approximately $23,853, the household's income as a share of the federal poverty guideline (school_meal_fpg_ratio) is 1.49, placing it above the reduced-price eligibility threshold of 1.85 times the federal poverty line. However, California's universal free school meals program provides free meals to all students regardless of income or categorical eligibility status, so the household's school_meal_tier is classified as FREE rather than reduced-price. Since the household already qualifies for the superior free meals tier through the state's universal program, it is not separately eligible for the reduced-price tier, resulting in a school_meal_net_subsidy of 0 for the reduced-price category.",
us,scenario_072,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Michigan household in 2026 because neither spouse reported self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this joint household's approximately $73,974 in total income derives from wages or other non-self-employment sources, there is no self-employment income base upon which to calculate the 15.3% self-employment tax rate (12.4% for Social Security and 2.9% for Medicare). Consequently, both individual entities in the household show $0 self-employment tax liability.",
us,scenario_089,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this North Carolina household is eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of [False, False]. WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this household contains only two adults with no children. Since the household composition does not include any individuals in the eligible age or demographic categories, PolicyEngine correctly classified both household members as ineligible for the program.",
-us,scenario_110,federal_income_tax_before_refundable_credits,23897.443359375,65,"PolicyEngine calculated the federal income tax before refundable credits as $23,897.44 for this single Ohio resident by first determining the adjusted gross income of $163,421.64, which included $95,755.10 in employment income (after $4,244.90 in pre-tax 401(k) contributions), $30,869.77 in capital gains, $21,301 in taxable interest income, and $15,694.12 in dividend income. After applying above-the-line deductions of $198.35 and taxable income deductions of $17,752.06 (including a $1,264.59 qualified business income deduction), the taxable income was reduced to $145,669.58. The tax was then computed in two parts: ordinary income tax at main rates of $17,764.88 applied to the ordinary taxable income, plus capital gains tax of $6,132.56 applied to the $40,883.77 in net capital gains and qualified dividends, with no net investment income tax owed since the household's modified adjusted gross income fell below the threshold for that tax.",
+us,scenario_110,federal_income_tax_before_refundable_credits,25747.7734375,71,"PolicyEngine calculated federal income tax before refundable credits of $25,747.77 for this household in 2026, combining $19,615.21 of main-rate tax and $6,132.56 of capital gains tax. IRS gross income and AGI are both $169,942.94. Income inputs include $95,755.10 of IRS employment income after pre-tax contributions, $30,869.77 of capital gains, $21,301.00 of taxable interest, $15,694.12 of dividends, and $6,322.94 of estate income. Selected deductions of $16,487.47, comprising the $16,100.00 standard deduction and $387.47 charitable deduction, leave taxable income of $153,455.47. The engine excludes $40,883.77 from main-rate income, leaving $112,571.70. The excluded amount comprises $30,851.77 of long-term capital gains and $10,032.00 of qualified dividends and feeds the separate capital gains calculation. Net investment income tax, other listed additional taxes, and capped non-refundable credits are zero.",
us,scenario_089,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False. WIC eligibility requires the presence of children under age 5 in the household, and this benchmark household contains 0 children. Since the household does not meet the basic categorical requirement for WIC participation, the spouse's eligibility status is False regardless of other factors such as the household's income of approximately $115,866.",
us,scenario_110,federal_refundable_credits,0.0,6,"For this single adult in Ohio with approximately $175,142 in household income, PolicyEngine calculated federal refundable credits of $0 for tax year 2026. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. This outcome reflects that the household's income level and composition do not qualify for any of these refundable tax benefits under 2026 tax law.",
us,scenario_013,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. The head of household is 80 years old, which exceeds CHIP's age eligibility requirements that typically cap at 18 or 19 years old depending on the state. Additionally, even if the age criterion were met, the household income of approximately $30,472 would need to fall within CHIP's income thresholds, which vary by state but are generally set at or below 200-250% of the federal poverty level. The combination of the person's advanced age and the program's age restrictions places them outside CHIP's eligible population, resulting in the False determination.",
@@ -968,7 +968,7 @@ us,scenario_072,state_refundable_credits,0.0,46,"PolicyEngine calculated state r
us,scenario_110,self_employment_tax,0.0,1,"PolicyEngine calculated a self_employment_tax value of $0 for this Ohio household because the household's income of approximately $175,142 was classified as W-2 wage income rather than self-employment income. Self-employment tax only applies to individuals with net earnings from self-employment activities, which are reported on Schedule C (Form 1040) and subject to the 15.3% combined Social Security and Medicare tax rate. Since this benchmark household had no self-employment income to report, PolicyEngine correctly determined that no self-employment tax obligation existed, resulting in a value of $0.",
us,scenario_013,free_school_meals_eligible,0.0,14,"PolicyEngine determined that this Arizona household is not eligible for free school meals, resulting in a value of 0. The household's countable income of $30,472—derived from social security ($23,736), pension income ($6,720), and dividend income ($16)—was compared against the federal poverty guideline of $15,960, yielding a school_meal_fpg_ratio of 1.91. Since this ratio exceeds the 185% threshold required for reduced-price meals (1.85), and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is assigned to school_meal_tier 2, which corresponds to full-price status. Although PolicyEngine calculated a potential daily subsidy of $0.87 for reduced-price meals, the school_meal_net_subsidy is set to 0 because the household's income disqualifies it from any meal assistance program.",
us,scenario_089,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in North Carolina for tax year 2026. The household's income of approximately $115,866 exceeds CHIP's income thresholds, which are set at a percentage of the federal poverty level and vary by household composition. Additionally, CHIP is primarily designed for children and pregnant women, and the head of household is a 47-year-old adult who does not fall within CHIP's covered age categories. Since the person fails both the income requirement and the categorical eligibility criteria (age), PolicyEngine correctly returned is_chip_eligible as False.",
-us,scenario_110,state_income_tax_before_refundable_credits,4057.47216796875,62,"PolicyEngine calculated the state income tax before refundable credits for this Ohio resident by first determining their Ohio adjusted gross income (AGI) of $163,421.64. From this AGI, PolicyEngine subtracted the Ohio personal exemption of $1,900, resulting in a taxable income of $161,521.64. This taxable income was then subject to Ohio's tax brackets and rates, which produced a tax liability of $4,057.47 before any refundable credits. All other states returned zero tax liability since the household resides in Ohio, making the Ohio calculation the sole contributor to the overall state income tax before refundable credits figure.",
+us,scenario_110,state_income_tax_before_refundable_credits,4236.8076171875,69,"PolicyEngine calculated Ohio income tax before refundable credits of $4,236.81 for this household in 2026. IRS gross income and person-level AGI are both $169,942.94. This amount flows through the person-level Ohio AGI calculation into Ohio AGI of $169,942.94. Subtracting $1,900.00 of Ohio personal exemptions gives taxable income of $168,042.94, also reported as taxable nonbusiness income. Tax on this income is $4,236.81 before non-refundable credits, and Ohio income tax before refundable credits remains $4,236.81. Every other state’s component shown is zero, so aggregate state income tax before refundable credits is also $4,236.81.",
us,scenario_052,federal_income_tax_before_refundable_credits,104211.40625,50,"PolicyEngine calculated the federal income tax before refundable credits as $104,211.41 for this Texas joint filer by starting with an adjusted gross income of $504,778.56, which included $499,073.84 in employment income (after $926.16 in pre-tax 401(k) contributions), $7,900 in taxable interest income, $848 in dividend income, and a net capital loss of $29,922. After applying above-the-line deductions of $3,043.28 and the standard deduction of $32,200, the household's taxable income was $472,578.56. The main income tax on this amount using 2026 tax brackets was $103,865.78, to which PolicyEngine added $127.20 in capital gains tax on the $848 of qualified dividends and $218.42 in net investment income tax (calculated on $5,748 of net investment income subject to the 3.8% NIIT threshold), resulting in a total income tax before refundable credits of $104,211.41.",
us,scenario_032,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0.00 for this Minnesota household in 2026 because neither adult in the household reported self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this joint household's income of approximately $42,664 derives from other sources (such as wages or salaries), there is no self-employment income subject to the 15.3% self-employment tax rate. The computation trace shows zero values across all household entities, confirming that no self-employment activity was present in the household's tax profile.",
us,scenario_013,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $30,472 translates to a school_meal_fpg_ratio of 1.91, meaning income is 191% of the federal poverty guideline. Since this ratio exceeds the 185% income threshold required for reduced-price meal eligibility, and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is classified in the school_meal_tier as PAID. Consequently, the school_meal_net_subsidy is 0, and the household must pay the full price for school meals.",
@@ -1136,7 +1136,7 @@ us,scenario_033,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employ
us,scenario_111,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not CHIP-eligible (is_chip_eligible = False) for the 2026 tax year in Washington State. The spouse, age 49, fails to meet CHIP's eligibility criteria on the basis of age alone, as CHIP covers children and pregnant women, with eligibility typically ending at age 19 (or slightly older for certain categories). Since the spouse is 49 years old, they fall outside CHIP's age-restricted coverage group regardless of income or other factors. Additionally, the engine confirmed the spouse is not Medicaid-eligible (category: NONE), which would have been an alternative pathway to coverage. Therefore, PolicyEngine correctly returned False for spouse_chip_eligible.",
us,scenario_091,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Wisconsin single-adult household with approximately $35,982 in income. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with each state-specific variable (al_refundable_credits, ar_refundable_credits, and so on through wi_refundable_credits) evaluated individually. For this household, wi_refundable_credits evaluated to $0, and all other state refundable credit variables similarly returned $0, resulting in a total state_refundable_credits value of $0. This outcome reflects that Wisconsin does not currently offer refundable tax credits that this household qualifies for, or the household's income and circumstances do not meet the eligibility thresholds for any available state-level refundable credits.",
us,scenario_073,free_school_meals_eligible,0.0,296,"PolicyEngine determined that this Michigan household is not eligible for free school meals, resulting in a value of 0.0. Although the household meets multiple pathways to eligibility—including Michigan's universal free school meals program and categorical eligibility through SNAP receipt of $287.68—the engine's school_meal_tier variable evaluated to 0 (not FREE), which directly set free_school_meals to 0. The household's income of $25,753.59 represents 1.61 times the federal poverty guideline, placing it above the standard income threshold for federal free meal eligibility. Despite qualifying for categorical eligibility and residing in a state with universal free school meals, the computation trace shows school_meal_tier as 0, indicating the household does not receive the free meals benefit under PolicyEngine's 2026 rules for this benchmark scenario.",
-us,scenario_015,local_income_tax,0.0,1,"For a single adult in Indiana with a household income of approximately $43,450 in tax year 2026, PolicyEngine calculated a local income tax liability of $0. This result reflects that Indiana does not impose a local income tax at the state level, and the household's residence does not have an applicable municipal or county income tax that would generate a tax obligation. Consequently, with no local income tax base to apply and no local tax rates in effect for this jurisdiction, the local income tax value defaults to zero.",
+us,scenario_015,local_income_tax,670.5501098632812,34,"PolicyEngine calculated local income tax of $670.55 using Adams County, Indiana, and Indiana AGI of $41,909.38. Employment income of $45,000.00 less $1,041.93 of pre-tax 401(k) contributions gives IRS employment income of $43,958.07. Adding $2,000.00 of taxable interest gives IRS gross income of $45,958.07. The household has a $4,510.00 long-term capital loss, of which the calculation deducts $3,000.00. Together with a $48.69 traditional IRA deduction, above-the-line deductions total $3,048.69, leaving federal AGI of $42,909.38. Indiana exemptions of $1,000.00 reduce this to Indiana AGI of $41,909.38. The county tax is $670.55, and all other listed local income taxes are zero.",
us,scenario_073,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, along with certain disability or end-stage renal disease criteria. For this single-adult Michigan household with an annual income of approximately $25,754, the head of household does not meet the age threshold or other qualifying conditions for Medicare enrollment. Consequently, PolicyEngine assigned a value of 0.0 (False) to the is_medicare_eligible variable, indicating no Medicare eligibility status for this benchmark household.",
us,scenario_053,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this 25-year-old single adult in 2026 because the household head does not meet SSI's eligibility criteria. SSI requires beneficiaries to be aged 65 or older, blind, or disabled; the engine's inputs indicate `is_ssi_aged_blind_disabled=False` for the head of household, meaning none of these conditions apply. Although the engine determined that the household would take up SSI if eligible (`takes_up_ssi_if_eligible=True`), the absence of a qualifying status—rather than income limits or other factors—is the binding constraint. Consequently, no SSI benefit was computed for this household.",
us,scenario_091,local_income_tax,0.0,1,"For a single adult in Wisconsin with a household income of approximately $35,982 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that Wisconsin does not impose a local income tax at the state level, and the household's income and circumstances do not trigger any local tax obligations in the jurisdiction modeled. The computation therefore directly returns zero, as there are no applicable local income tax provisions to apply to this household's income.",
@@ -1304,7 +1304,7 @@ us,scenario_075,head_chip_eligible,0.0,2,"PolicyEngine determined that the head
us,scenario_114,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Virginia household in 2026. The household, consisting of a single adult with no children and an annual income of approximately $110,072, does not qualify for TANF assistance. TANF is a needs-based program primarily designed to support families with dependent children, and this household's composition and income level place it well above the eligibility thresholds for the program. Consequently, PolicyEngine assigned a benefit value of zero.",
us,scenario_054,child1_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this North Carolina household are eligible for Head Start in 2026, resulting in an eligibility array of [False, False, False]. Head Start eligibility is typically determined by federal income thresholds, which are generally set at or below 100% of the federal poverty line, though states may have additional criteria. Given that this household has approximately $0 in annual income, the ineligibility result suggests that PolicyEngine applied additional eligibility requirements beyond income alone—such as age restrictions (Head Start typically serves children ages 3-5), enrollment status, or other program-specific criteria that were not met by the children in this benchmark household. The three-element array structure indicates PolicyEngine evaluated eligibility for the household head and both children, with all three returning False.",
us,scenario_036,free_school_meals_eligible,0.0,12,"PolicyEngine determined that this New Jersey household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_fpg_ratio of 3.71 indicates that their countable income of $59,220 is 3.71 times the federal poverty guideline of $15,960 for their household size. Since this ratio exceeds the 185% threshold required for reduced-price meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they are classified in school_meal_tier 2 (PAID). New Jersey does not offer universal free school meals, so no state-level program provides an alternative pathway to eligibility. Consequently, the household must pay the full price for school meals with no subsidy.",
-us,scenario_018,state_income_tax_before_refundable_credits,1146.05224609375,64,"PolicyEngine calculated Arizona state income tax before refundable credits of $1,146.05 for this single filer with ~$68,055 household income. The calculation began with an adjusted gross income (AGI) of $61,592.09, from which Arizona deductions of $15,750 were subtracted to arrive at taxable income of $45,842.09. The deduction amount was determined by comparing itemized deductions of $10,786.71 (which included $1,286.71 in state and local taxes and $9,500 in medical expenses) against the standard deduction of $15,750, with the standard deduction selected as the larger amount. Arizona's tax rate schedule was then applied to the taxable income of $45,842.09 to produce the final tax liability of $1,146.05 before any refundable credits.",
+us,scenario_018,state_income_tax_before_refundable_credits,1137.30224609375,72,"PolicyEngine calculated Arizona income tax before refundable credits of $1,137.30 for this single filer in 2026. Arizona adjusted gross income is $61,592.09. Arizona's 2026 standard deduction for a single filer is $16,100: the $15,750 HB 4168 set, indexed as the federal basic standard deduction is (A.R.S. 43-1041(H)). It exceeds the household's $10,637.30 of itemized deductions, so taxable income is $45,492.09, and Arizona's 2.5% rate gives $1,137.30.",
us,scenario_093,state_income_tax_before_refundable_credits,3388.24560546875,95,"PolicyEngine calculated Missouri income tax before refundable credits of $3,388.25 for this married couple filing jointly, who claim an adult dependent. The spouses' Missouri adjusted gross income is $112,408.25 ($48,955.40 and $63,452.85) after 401(k) deferrals and above-the-line deductions; the dependent's Missouri adjusted gross income is $0 in the engine's calculation, so that person owes no Missouri tax here. The couple takes the $32,200 standard deduction, larger than Missouri itemized deductions of $12,469.50, and a $431.45 deduction for federal income tax, leaving Missouri taxable income of $79,776.80, allocated between the spouses in proportion to income ($34,743.94 and $45,032.86). Missouri's 2026 brackets, as the Department of Revenue published them in November 2025, give $1,452.33 and $1,935.91, a total of $3,388.25.",
us,scenario_075,head_medicare_eligible,0.0,1,"PolicyEngine determined that the household head is not Medicare eligible in 2026, resulting in a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65. Since this benchmark household consists of a single adult with a household income of approximately $109,286 and no indication of disability or end-stage renal disease status, the household head does not meet the age threshold required for Medicare enrollment. Therefore, PolicyEngine correctly classified the head of household as ineligible for Medicare benefits in the 2026 tax year.",
us,scenario_114,head_wic_eligible,0.0,17,"PolicyEngine determined that this household is not eligible for WIC benefits (is_wic_eligible = False) because it failed the income test, despite meeting the nutritional risk criteria. The household's wic_countable_income of $85,732—derived from employment income ($22,700), social security ($23,328), pension income ($30,000), retirement distributions ($9,600), and interest income ($104)—substantially exceeds the wic_income_limit of $29,526 for a single-person household in the contiguous United States. Additionally, the household does not meet categorical eligibility requirements, as the head of household is 69 years old, placing them in WIC category 5, which does not qualify for the program. Although PolicyEngine identified that the household meets nutritional risk criteria (is_wic_at_nutritional_risk = True), this alone is insufficient for WIC eligibility without satisfying both the income test and categorical requirements.",
@@ -1352,7 +1352,7 @@ us,scenario_114,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin
us,scenario_115,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household with approximately $23,442 in annual income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. For a single adult with no children and this income level, the household does not qualify for the EITC (which requires either dependent children or specific age/income thresholds for childless workers), the refundable CTC (which requires qualifying children), or the other refundable credits evaluated.",
us,scenario_018,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in Arizona for 2026, resulting in an is_chip_eligible value of False. The determination is based on chip_category = 3, which indicates the person fails to meet CHIP's eligibility criteria. Since CHIP is a program designed for children and teenagers, and this household consists of a single adult age 53, the person exceeds the maximum age requirement for CHIP enrollment. Additionally, with a household income of approximately $68,055, the person's income level places them outside CHIP's income-based eligibility thresholds for the applicable category. The combination of age ineligibility and income ineligibility results in the False determination.",
us,scenario_002,reduced_price_school_meals_eligible,0.0,3,"This household is not eligible for reduced-price school meals, resulting in a value of 0. With a household income of approximately $134,820, the income-to-federal poverty guideline ratio is 5.12, which far exceeds the 185% threshold required for reduced-price meal eligibility. Washington State does not offer universal free school meals, and the household does not qualify for categorical eligibility through programs like SNAP or TANF. Consequently, PolicyEngine assigns the household a school_meal_tier of PAID, meaning they must pay the full price for school meals, and the school_meal_net_subsidy is 0.",
-us,scenario_076,state_income_tax_before_refundable_credits,6818.34423828125,116,"PolicyEngine calculated Idaho income tax before refundable credits of $6,818.34 for this head-of-household filer. Federal adjusted gross income of $164,220 less Idaho's $1,000 household and dependent care deduction gives Idaho adjusted gross income of $163,220. Subtracting the $24,150 standard deduction and an $800 qualified business income deduction on $4,000 of qualified business income leaves Idaho taxable income of $138,270. Idaho taxes income above the head-of-household zero-rate threshold of $9,622, the 2025 amount the Idaho State Tax Commission published and the last located before the benchmark's 2026-07-03 reference freeze, at 5.3%: ($138,270 - $9,622) x 5.3% = $6,818.34.",
+us,scenario_076,state_income_tax_before_refundable_credits,6828.34423828125,120,"PolicyEngine calculated Idaho state income tax before refundable credits of $6,828.34 for this head-of-household filer in 2026. Federal adjusted gross income is $164,220. Less the $24,150 standard deduction, $1,000 of Idaho subtractions and an $800 qualified business income deduction, Idaho taxable income is $138,270, and Idaho income tax is $6,818.34, with no nonrefundable credits. State income tax adds Idaho's $10 permanent building fund tax, for $6,828.34.",
us,scenario_037,local_income_tax,0.0,1,"For a single adult in North Carolina with a household income of approximately $41,943 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that North Carolina does not impose a local income tax at the county or municipal level—only the state income tax applies to residents' earned income. Since there is no local income tax obligation in the jurisdiction, the local_income_tax variable correctly returns zero regardless of the household's income level.",
us,scenario_093,dependent1_wic_eligible,0.0,1,"PolicyEngine determined that none of the three household members are eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False, False]`. WIC eligibility requires applicants to meet specific categorical requirements—typically being pregnant, postpartum, breastfeeding, or a child under age 5—combined with income limits set at 185% of the federal poverty line. Since this household consists of 3 adults with no children and a household income of approximately $164,156, the members do not satisfy the categorical requirements for WIC participation. Therefore, PolicyEngine correctly computed that dependent1_wic_eligible equals 0.0, indicating no eligible household members.",
us,scenario_115,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this Alabama household as $0. The computation shows that the employee_payroll_tax component, which represents Social Security and Medicare taxes withheld from wages, evaluated to $0. Since the household is identified as a tax unit head (is_tax_unit_head = True), this individual is treated as the primary taxpayer for payroll tax purposes. Despite the household having an income of approximately $23,442, which would typically be subject to payroll taxes, the employee_payroll_tax calculation resulted in no tax liability, leading to the final payroll tax value of $0.",
@@ -1360,7 +1360,7 @@ us,scenario_055,payroll_tax,6398.08837890625,5,"PolicyEngine calculated a payrol
us,scenario_018,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65. Since this benchmark household is a single adult with a household income of approximately $68,055 and no indication of qualifying disability or end-stage renal disease status, the household member does not meet the age threshold required for Medicare enrollment. Therefore, PolicyEngine correctly assigned a value of 0.0 (False) to the is_medicare_eligible variable, indicating no Medicare eligibility for this household.",
us,scenario_037,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this single-adult household in North Carolina with an annual income of approximately $41,943. Although the model determined that the household would take up SNAP benefits if eligible (takes_up_snap_if_eligible = True), the household's income level exceeds the gross income limit for SNAP eligibility in 2026, resulting in no benefit entitlement. The repeated computation nodes in the trace reflect PolicyEngine's systematic evaluation across different benefit calculation pathways, all converging on the same zero-benefit outcome due to the income threshold constraint.",
us,scenario_055,self_employment_tax,0.0,1,"PolicyEngine calculated a self_employment_tax value of $0 for this Florida household because the household's income structure does not include self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, which arise when individuals operate their own business or are self-employed. Since this benchmark household has a total income of approximately $157,137 derived from sources other than self-employment (such as wages or other non-self-employment income), there is no self-employment income base upon which to calculate the 15.3% self-employment tax rate (12.4% for Social Security and 2.9% for Medicare). Therefore, PolicyEngine correctly assigned a self_employment_tax value of $0.",
-us,scenario_003,federal_income_tax_before_refundable_credits,22154.69921875,50,"PolicyEngine calculated the federal income tax before refundable credits for this Texas joint filer by starting with an IRS gross income of $184,872.25, which included $94,720 in taxable retirement distributions, $50,583.25 in taxable Social Security benefits, $32,525.29 in taxable pension income, $4,304 in dividend income, and $2,739.71 in taxable interest income, partially offset by $1,582 in capital losses. After applying above-the-line deductions of $2,699.98 (primarily from the capital loss deduction and traditional IRA contributions), the adjusted gross income reached $182,172.27. The household claimed the standard deduction of $32,200 rather than itemizing, resulting in a taxable income of $149,972.27. The income tax on this amount using the 2026 tax brackets for married filing jointly was $21,590.70, to which PolicyEngine added $564 in capital gains tax on the $3,760 of net capital gains taxed at preferential rates, yielding a total federal income tax before refundable credits of $22,154.70.",
+us,scenario_003,federal_income_tax_before_refundable_credits,22400.654296875,76,"PolicyEngine calculated federal income tax before refundable credits of $22,400.65 for this joint filer household. IRS gross income of $184,872.25 comprises $94,720.00 of taxable retirement distributions, $50,583.25 of taxable Social Security, $32,525.29 of taxable pension income, $4,304.00 of dividends, and $2,739.71 of taxable interest. A $1,582.00 above-the-line loss deduction reduces AGI to $183,290.25, and the $32,200.00 standard deduction reduces taxable income to $151,090.25. The calculation taxes $3,760.00 of qualified dividends at preferential rates, outside the main-rate calculation. Main-rate tax of $21,836.65 plus capital gains tax of $564.00 gives $22,400.65. Modified AGI is $183,290.25, and net investment income tax is $0.00. The listed additional taxes, penalties, and capped non-refundable credits are also zero.",
us,scenario_003,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household of two adults with no children and approximately $197,459 in household income. The computation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. The household's income level and family composition—specifically the absence of qualifying children and income above EITC phase-out thresholds for joint filers—rendered them ineligible for these refundable credits in tax year 2026.",
us,scenario_037,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this household because the head of household, age 44, does not meet any of SSI's eligibility criteria. SSI is restricted to individuals who are aged 65 or older, blind, or disabled; since the household member is is_ssi_aged_blind_disabled=False, they are ineligible for the program. Although the computation trace shows takes_up_ssi_if_eligible=True, indicating the household would claim SSI if eligible, the absence of a qualifying condition means no benefit amount is calculated. The $0.00 result reflects ineligibility based on age and disability status, not income limits or other factors.",
us,scenario_018,free_school_meals_eligible,0.0,13,"PolicyEngine determined that this Arizona household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_fpg_ratio of 4.26—calculated by dividing countable income of $68,055 by the federal poverty guideline of $15,960—places them in school_meal_tier 2 (paid meals). Since Arizona does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, their income exceeds the 185% reduced-price limit threshold. Consequently, the household pays full price for school meals with no subsidy applied.",
@@ -1605,7 +1605,7 @@ us,scenario_077,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin
us,scenario_057,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this Louisiana household is eligible for WIC (Women, Infants, and Children) benefits in 2026, resulting in a value of False for both household members. WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this benchmark household consists of two adults with no children. Since the household contains no pregnant women, postpartum women, breastfeeding women, infants, or young children, neither member meets the categorical requirements for WIC participation regardless of their income level of approximately $38,000.",
us,scenario_117,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Arkansas household in 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with ar_refundable_credits specifically evaluated for Arkansas (the household's state of residence) returning $0. All other state-level refundable credit variables—including those for Alabama, Arizona, California, Colorado, and all remaining states—also evaluated to $0. Since every component of the state refundable credits calculation yielded zero, the total state_refundable_credits value is $0, indicating this household qualifies for no refundable state tax credits in 2026.",
us,scenario_038,free_school_meals_eligible,1.0,258,"PolicyEngine determined that this Louisiana household qualifies for free school meals with an annual benefit value of $2,261.92. The household meets eligibility through categorical qualification via SNAP receipt ($7,286.94 annually), which automatically confers free meal status regardless of income thresholds. Additionally, the household's income of $22,992.50 represents 70% of the federal poverty guideline (school_meal_fpg_ratio of 0.70), placing it well below the 130% income threshold required for free meals. With two children in K-12 school (is_in_k12_school showing True for both children), the household receives a daily subsidy of $7.15 per child, yielding the total annual benefit of $2,261.92 after accounting for the school meal paid daily subsidy of $0.87.",
-us,scenario_022,state_income_tax_before_refundable_credits,2439.650146484375,75,"PolicyEngine calculated California state income tax before refundable credits of $2,439.65 for this single California resident with ~$112,259 household income. The calculation began with an adjusted gross income of $109,149.16, from which California-specific AGI subtractions of $17,624.75 (primarily taxable social security) were deducted to arrive at California AGI of $91,524.41. From this AGI, California deductions totaling $23,676.67 were subtracted—comprising a standard deduction of $5,706 and itemized deductions of $23,676.67 (which included $14,847.67 in charitable contributions and $8,829 in real estate taxes)—resulting in taxable income of $67,847.75. This taxable income was then subject to California's tax rate schedule, producing a tax before credits of $2,752.58, which was reduced by non-refundable credits of $312.93 (California exemptions based on the household member's aged/blind status) to yield the final tax liability of $2,439.65.",
+us,scenario_022,state_income_tax_before_refundable_credits,1968.805419921875,87,"PolicyEngine calculated California income tax before refundable credits of $1,968.81 for this household with a 77-year-old head. Federal AGI of $109,149.16 is reduced by $17,624.75 of taxable Social Security to California AGI of $91,524.41. Itemized deductions total $30,389.97: $8,829.00 of real estate taxes, $15,393.41 of charitable deductions, and $6,167.56 of miscellaneous deductions. The charitable amount equals $14,693.41 of cash donations plus $700.00 of non-cash donations. Itemized deductions exceed the $5,706.00 California standard deduction. Subtracting them from California AGI gives taxable income of $61,134.44. Tax before credits is $2,274.81; the $306.00 exemption credit, calculated with an aged/blind count of one, gives the final $1,968.81.",
us,scenario_078,federal_income_tax_before_refundable_credits,24164.45703125,35,"PolicyEngine calculated federal income tax before refundable credits of $24,164.46 for this Maryland single filer by applying 2026 tax rates to a taxable income of $131,526.91. The taxable income was derived from an adjusted gross income of $200,136.91—which included $198,505 in employment income plus $1,631.91 in state and local tax refund income—reduced by $68,610 in itemized deductions. The household itemized rather than taking the standard deduction of $16,100, claiming $40,400 in state and local tax deductions and $28,210 in interest deductions, which resulted in a lower tax liability of $24,164.46 compared to $36,766.86 if the standard deduction had been used. No additional taxes or credits adjusted this amount, as net investment income tax and other components were zero.",
us,scenario_117,local_income_tax,0.0,1,"PolicyEngine calculated a local_income_tax value of $0.00 for this Arkansas household in 2026. Arkansas does not impose a local income tax on residents, which is why the local_income_tax variable resolves to zero regardless of the household's income level of approximately $250,733. This result reflects the state's tax structure, where income taxation occurs only at the state and federal levels, with no additional local income tax liability assessed by municipalities or counties.",
us,scenario_098,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. CHIP eligibility requires meeting both age and income thresholds, and this 80-year-old household member fails CHIP's age criterion, as the program is designed for children and young adults, not seniors. With a household income of approximately $35,035 and no dependent children in the household, the person does not qualify under any CHIP category. The resulting chip_category of 3 reflects ineligibility, leading to the final determination that is_chip_eligible equals False.",
@@ -1619,7 +1619,7 @@ us,scenario_078,federal_refundable_credits,0.0,6,"PolicyEngine calculated federa
us,scenario_001,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Virginia household of two adults with approximately $44,289 in annual income for tax year 2026. Although the household's income level would typically qualify for SNAP eligibility, the computation trace shows that `takes_up_snap_if_eligible` was set to True across all calculation periods, indicating the household was modeled as willing to participate in the program. However, the household's income of $44,289 exceeds the gross income limit for SNAP eligibility for a two-person household in 2026, resulting in a $0 benefit despite the assumed take-up behavior. The repeated entries in the dependency tree reflect PolicyEngine's monthly calculation structure, with each month independently confirming zero benefits due to the income threshold being exceeded.",
us,scenario_117,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Arkansas household in 2026. Although the household indicated a willingness to take up SNAP benefits if eligible (takes_up_snap_if_eligible = True), the household's annual income of approximately $250,733 substantially exceeds the income eligibility threshold for SNAP. With a household size of five members (two adults and three children), the income limit for SNAP eligibility in Arkansas is significantly lower than this household's income level, resulting in automatic disqualification. Consequently, PolicyEngine determined that despite the household's stated participation preference, no SNAP benefit could be awarded due to income ineligibility.",
us,scenario_022,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this California household in 2026. California does not impose a local income tax at the state level, and while some California municipalities may have local taxes on specific types of income (such as gross receipts taxes or business taxes), these do not apply to ordinary wage and salary income for individual residents. For a single adult with approximately $112,259 in household income earning standard employment income, no local income tax liability was triggered under California's tax code.",
-us,scenario_039,federal_income_tax_before_refundable_credits,1345.510498046875,41,"PolicyEngine calculated federal income tax before refundable credits of $1,345.51 for this single Virginia filer in 2026 by applying the standard tax rate schedule to a taxable income of $13,279.25. The taxable income was derived from an adjusted gross income of $32,699.07, which included $10,129.10 in taxable Social Security benefits, $26,800 in taxable retirement distributions, and $2,030 in taxable pension income, reduced by $6,260.03 in above-the-line deductions for self-employment losses. From this adjusted gross income, PolicyEngine applied $19,419.81 in deductions, consisting of the $16,100 standard deduction plus a $3,319.81 qualified business income deduction (calculated as 20% of $16,599.07 in qualified business income). No additional taxes or credits affected the final calculation, as net investment income tax, recapture of investment credit, and other adjustments all equaled zero.",
+us,scenario_039,federal_income_tax_before_refundable_credits,8596.0283203125,31,"PolicyEngine calculated federal income tax before refundable credits of $8,596.03 for this household in 2026. IRS gross income is $85,469.25, comprising $30,689.25 of taxable Social Security benefits, $26,800.00 of taxable retirement distributions, $2,030.00 of taxable pension income, and $25,950.00 of estate income. A self-employment loss of $6,260.03 is deducted above the line, reducing AGI to $79,209.22. The $16,100.00 standard deduction exceeds the $3,793.91 itemized taxable-income deduction and is selected, leaving taxable income of $63,109.22. Main-rate tax is $8,596.03. Estate income of $25,950.00 also supplies net investment income, but net investment income tax is $0.00. The other listed additions and capped non-refundable credits are zero, leaving $8,596.03.",
us,scenario_098,free_school_meals_eligible,0.0,13,"PolicyEngine determined that this Illinois household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_countable_income of $46,135, derived from social security ($30,595) and pension income ($15,540), was compared against the federal poverty guideline threshold. With a school_meal_fpg_ratio of 2.89—meaning the household's income is 289% of the federal poverty guideline of $15,960—the household exceeds the 185% income limit for reduced-price meals. Since the state does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is classified in the PAID school_meal_tier and receives no subsidy.",
us,scenario_078,payroll_tax,14317.322265625,5,"PolicyEngine calculated a payroll tax of $14,317.32 for this single Maryland resident with approximately $200,137 in household income. This total comprises two components: employee Social Security tax of $11,439 and employee Medicare tax of $2,878.32. The Social Security tax reflects the 6.2% employee contribution rate applied to earnings up to the 2026 wage base limit, while the Medicare tax reflects the 1.45% employee contribution rate applied to total wages without a cap. The calculation identifies the household member as a tax unit head, which determines the applicable payroll tax obligations for this individual filer.",
us,scenario_117,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) for this Arkansas household in 2026 as $0.00 because none of the five household members—the head (age 44), spouse (age 34), or the three children (ages 16, 11, and 2)—meet SSI's eligibility criteria. SSI requires that a person be aged 65 or older, blind, or disabled; the engine's inputs indicate that all household members have `is_ssi_aged_blind_disabled=False`, meaning none qualify on any of these grounds. Although the engine confirmed that all members have `takes_up_ssi_if_eligible=True` (indicating willingness to claim if eligible), this take-up assumption is moot because no eligibility threshold was met. Consequently, each individual received $0.00 in SSI benefits, yielding a household total of $0.00.",
@@ -1637,7 +1637,7 @@ us,scenario_078,state_income_tax_before_refundable_credits,6936.87646484375,87,"
us,scenario_117,head_wic_eligible,0.0,1,"PolicyEngine determined that none of the five household members are eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False, False, False, False]`. WIC eligibility requires that applicants meet categorical requirements (being pregnant, postpartum, breastfeeding, an infant, or a child under age 5) and satisfy income limits set at 185% of the federal poverty line. With a household income of approximately $250,733 annually in Arkansas, this household substantially exceeds the WIC income threshold, which for a family of five would be roughly $50,000–$55,000 depending on the specific composition and poverty guidelines applied. The high household income alone disqualifies all members from WIC participation, regardless of whether any household members meet the categorical requirements for the program.",
us,scenario_022,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this California household in 2026. The household consists of a single adult with no children and an annual income of approximately $112,259. TANF eligibility is restricted to families with dependent children, and since this household has zero children, it does not qualify for the program regardless of income level. Therefore, PolicyEngine correctly determined that no TANF benefit applies to this household.",
us,scenario_039,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Virginia household in 2026 because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $84,625 in household income derives from other sources (such as wages or salaries), there is no self-employment tax liability. The calculation correctly reflects that self-employment tax applies only to individuals who operate their own business or are self-employed, which does not apply to this household's income profile.",
-us,scenario_099,federal_income_tax_before_refundable_credits,10679.75,90,"PolicyEngine calculated the federal income tax before refundable credits as $10,679.75 by starting with an income tax before credits of $16,999.75 and subtracting $6,320 in non-refundable credits. The income tax before credits was derived from a taxable income of $125,385.05, which resulted from an adjusted gross income of $159,585.05 reduced by $34,200 in standard deductions; the AGI itself reflected $161,912.80 in employment income (after $3,087.20 in pre-tax 401(k) contributions) plus $1,010 in taxable interest and $144 in dividend income, partially offset by $6,478 in net capital losses and $3,481.76 in above-the-line deductions. The tax on this taxable income was calculated using ordinary income tax rates plus $19.20 in capital gains tax on $128 of qualified dividends. The non-refundable credits totaling $6,320 consisted of $1,920 in child and dependent care credits (based on $6,000 in eligible childcare expenses at a 32% rate) and $4,400 in non-refundable child tax credits for the two qualifying children.",
+us,scenario_099,federal_income_tax_before_refundable_credits,10711.484375,130,"PolicyEngine calculated federal income tax before refundable credits of $10,711.48 for this joint filer household in 2026. IRS gross income is $163,066.80, comprising $161,912.80 of IRS employment income after $3,087.20 of pre-tax contributions, $1,010.00 of taxable interest, and $144.00 of dividends. A $6,478.00 net capital loss is separately recorded. Above-the-line deductions of $3,337.50, comprising a $3,000.00 loss deduction and $337.50 educator expense deduction, give AGI of $159,729.30. The $32,200.00 standard deduction and $2,000.00 charitable deduction for non-itemizers leave taxable income of $125,529.30. After excluding $128.00 of qualified dividends, main-rate tax is $17,012.29. Capital gains tax is $19.20, and total tax before credits is $17,031.48. Non-refundable credits total $6,320.00: a $1,920.00 child and dependent care credit, calculated as 32% of $6,000.00 in expenses capped from $12,740.00, plus a $4,400.00 child tax credit. The final amount is $10,711.48.",
us,scenario_001,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this Virginia household is eligible for WIC (Women, Infants, and Children benefits) in 2026, resulting in an `is_wic_eligible` value of [False, False]. WIC eligibility requires applicants to be pregnant women, postpartum women, breastfeeding women, or caregivers of children under age 5. Since this benchmark household contains only two adults with no children, neither household member meets the categorical requirement for WIC participation. The household's income of approximately $44,289 is also above the WIC income threshold of 185% of the federal poverty line (roughly $39,000 for a household of two in 2026), which would further disqualify them even if they met the categorical requirements. Consequently, PolicyEngine correctly computed zero WIC eligibility across both household members.",
us,scenario_057,spouse_chip_eligible,0.0,1,"For the spouse in this Louisiana household, PolicyEngine determined that `is_chip_eligible` equals False because the spouse is already eligible for Medicaid as an adult. CHIP (Children's Health Insurance Program) is designed to cover children who do not qualify for Medicaid, so individuals who are Medicaid-eligible are automatically ineligible for CHIP regardless of their age or income. Since the spouse meets Louisiana's Medicaid eligibility criteria in the adult category, they fall outside the population CHIP is intended to serve, resulting in a CHIP eligibility value of False.",
us,scenario_078,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Maryland resident with approximately $200,137 in household income. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with md_refundable_credits specifically evaluated for this household's Maryland residency status. Since Maryland does not offer refundable tax credits that apply to this household's income level and filing status (single adult with no children), md_refundable_credits returned $0. All other state-level refundable credit variables similarly evaluated to $0, resulting in a total state_refundable_credits value of $0.",
@@ -1645,7 +1645,7 @@ us,scenario_099,federal_refundable_credits,0.0,6,"PolicyEngine calculated federa
us,scenario_022,head_wic_eligible,0.0,17,"PolicyEngine determined that this household is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False. The primary reason for ineligibility is that the household's wic_countable_income of $112,630 far exceeds the wic_income_limit of $29,526, causing the meets_wic_income_test to fail. Additionally, the household does not meet categorical eligibility requirements, as the head of household is 77 years old and falls into wic_category 5 (which does not qualify for WIC). Although PolicyEngine determined the household meets is_wic_at_nutritional_risk, this alone is insufficient to establish eligibility without passing both the income test and categorical eligibility criteria.",
us,scenario_117,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False for the 2026 tax year. The WIC program has strict income eligibility thresholds, typically set at 185% of the federal poverty line, and this household's income of approximately $250,733 substantially exceeds that limit. With a household income in the upper-middle range and three children, the family's earnings place them well above the income cutoff required to qualify for WIC assistance. Consequently, PolicyEngine's eligibility array shows False across all household members, indicating no WIC eligibility for the spouse or any other family member.",
us,scenario_038,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that child 2 is not eligible for Early Head Start in this Louisiana household for 2026, resulting in a value of False. Early Head Start eligibility is typically restricted to children under age 3, and this household's second child does not meet the age requirement for the program. The household's income of approximately $22,992 would generally qualify them based on federal poverty guidelines, which typically allow families at or below 100% of the federal poverty line to participate, but the child's age disqualifies them from this particular early intervention program. Therefore, PolicyEngine correctly assigned an eligibility status of False (0.0) for child 2.",
-us,scenario_039,state_income_tax_before_refundable_credits,514.4984130859375,83,"PolicyEngine calculated Virginia state income tax before refundable credits of $514.50 for this single adult in Virginia with approximately $84,625 household income. The calculation began with an adjusted gross income (AGI) of $32,699.07, which was reduced by $10,129.10 in Virginia subtractions (primarily taxable Social Security income), yielding a Virginia AGI of $22,569.97. From this amount, Virginia taxable income of $12,889.97 was determined by subtracting $8,750 in deductions (the standard deduction) and $930 in personal exemptions. The tax was then computed on this taxable income using Virginia's tax brackets and rates, resulting in the final tax liability of $514.50 before any refundable credits.",
+us,scenario_039,state_income_tax_before_refundable_credits,1975.7982177734375,80,"PolicyEngine calculated Virginia income tax before refundable credits of $1,975.80 for this single filer in 2026. Gross income is $85,469.25: $30,689.25 of taxable Social Security benefits, $26,800 of taxable IRA distributions, $2,030 of taxable pension income and $25,950 of estate and trust income, which federal gross income includes (26 U.S.C. 61(a)(14)). A $6,260.03 self-employment loss leaves federal adjusted gross income of $79,209.22. Virginia subtracts the taxable Social Security benefits, for Virginia adjusted gross income of $48,519.97. Less the $8,750 standard deduction and a $930 personal exemption, taxable income is $38,839.97, taxed at $1,975.80.",
us,scenario_057,head_medicare_eligible,0.0,1,"PolicyEngine determined that neither adult in this Louisiana household is Medicare eligible in 2026, resulting in a value of [False, False]. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, and this household's adults do not meet that age threshold. Given the household composition of two working-age adults with a combined income of approximately $38,000, PolicyEngine's calculation reflects that neither household member has reached the standard Medicare eligibility age. The reference value of 0.0 corresponds to the boolean False result, indicating no Medicare eligibility for either household member in the 2026 tax year.",
us,scenario_078,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Maryland household in 2026. Maryland does not impose a local income tax at the state level, and while some Maryland municipalities may have local tax provisions, PolicyEngine's standard calculation for Maryland residents does not apply local income taxes to household income in this tax year. Therefore, with no applicable local income tax jurisdiction or rate applying to this single adult earning approximately $200,137, the local income tax liability remained at zero.",
us,scenario_001,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not WIC-eligible (value: False) for this Virginia household in 2026. WIC (Women, Infants, and Children) eligibility requires the presence of qualifying dependents—specifically children under age 5 or pregnant/postpartum women—in the household. Since this benchmark household contains only 2 adults with 0 children, there are no qualifying individuals who could receive WIC benefits. Consequently, PolicyEngine correctly returned False for the spouse's WIC eligibility status, resulting in a reference value of 0.0 for any associated benefits.",
diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf
index 6da73bcd..bc7f7329 100644
Binary files a/app/public/paper/policybench.pdf and b/app/public/paper/policybench.pdf differ
diff --git a/app/public/paper/web/figures/positive_zero_scatter.png b/app/public/paper/web/figures/positive_zero_scatter.png
index 35745974..8bf8d3cd 100644
Binary files a/app/public/paper/web/figures/positive_zero_scatter.png and b/app/public/paper/web/figures/positive_zero_scatter.png differ
diff --git a/app/public/paper/web/index.html b/app/public/paper/web/index.html
index 25aa047e..1de6a8b4 100644
--- a/app/public/paper/web/index.html
+++ b/app/public/paper/web/index.html
@@ -151,7 +151,7 @@
Table of contents
Abstract
-
PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.15.17. It evaluates 46 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 86% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 71.1% weighted exact by 24.7 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-09-30), the top-scoring US model on exact match is GPT-6 Sol at 95.8%, while GPT-5.4 nano is lowest at 66.5%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 77.4% exact, below the 82.2% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 7,527 rows whose legacy threshold score is below 1. That universe contains 7,523 of the snapshot’s 7,523 exact-match misses and 4 exact hits; 2,072 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-09-29 with policyengine-us 2.15.17, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 28 outputs whose references rest on engine defects the audits found and upstream has not fixed, and the 36 whose references depend on an input the prompt does not state. The 0.7% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.
+
PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.38.6. It evaluates 47 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 86% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 69.9% weighted exact by 25.6 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-10-10), the top-scoring US model on exact match is GPT-6 Sol at 95.6%, while GPT-5.4 nano is lowest at 65.6%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 76.2% exact, below the 80.9% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 8,001 rows whose legacy threshold score is below 1. That universe contains 7,997 of the snapshot’s 7,997 exact-match misses and 4 exact hits; 2,138 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-10-10 with policyengine-us 2.38.6, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 10 outputs whose references rest on engine defects the audits found and upstream has not fixed, the 46 whose references depend on an input the prompt does not state, and two whose references rest on an amount a state published after the freeze. The 0.7% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.
Introduction
Household tax-and-benefit estimation sits between arithmetic and policy discussion. Each case has numeric labels, but generating those labels requires filing status, household composition, income concepts, program thresholds, and jurisdiction-specific rules. PolicyBench measures whether models can map household records to those outputs.
Most current language-model evaluations do not test this mapping directly. General math benchmarks emphasize symbolic manipulation and exact final answers (Cobbe et al. 2021; Hendrycks et al. 2021), while generic question-answering benchmarks emphasize recall or instruction following. PolicyBench instead asks whether models can transform household facts into policy outputs without access to tools or simulators.
@@ -163,11 +163,11 @@
Related work
The nearest public-benefits benchmark is the concurrent Public Benefits Bench, built by Vals AI with the Center for Civic Futures and Code for America (Vals AI 2026). It evaluates models on 459 expert-validated SNAP guidance scenarios (230 in the published test set) spanning all 50 US states plus Guam and the Virgin Islands, and grades free-text answers against expert-written rubric criteria using a judge model that agreed with SNAP policy experts on 80.6% of grading decisions. Where PolicyBench isolates parametric knowledge by withholding tools, Public Benefits Bench varies tool access directly: across models, multi-turn conversation raises pass rates by roughly 7.6 percentage points and web search by about 6.9. The two benchmarks probe complementary failure surfaces — conversational guidance quality for a single program with tools available, versus no-tool numeric estimation across tax and transfer outputs scored deterministically against microsimulation references — and reach consistent conclusions. The best model passes 71.7% of rubric criteria under Public Benefits Bench’s strongest condition, and its authors conclude that “no general-purpose AI model performs well enough to be trusted with SNAP benefits guidance,” matching the deployability gaps PolicyBench reports for unaided calculation.
Finally, PolicyBench depends on two infrastructure literatures that are not themselves large language model (LLM) benchmarks. One is structured-output reliability, since the benchmark relies on multi-output JSON responses and parse coverage rather than free-form prose (Shorten et al. 2024). The other is tax-benefit microsimulation, where systems such as EUROMOD provide the methodological precedent for evaluating policy rules over household microdata (Sutherland and Figari 2013). There is also operational work on applying and evaluating AI systems in public-benefits settings, including caseworker-assist and Supplemental Nutrition Assistance Program (SNAP)-focused evaluations (Nava Labs 2026, 2025; ZenML LLMOps Database 2025). PolicyBench combines these strands into a public cross-model benchmark over household-level tax and benefit outputs.
Benchmark design
-
The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 46 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.
+
The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 47 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.
Headline metric: exact match
The headline metric is the exact-match rate: the share of requested outputs the model gets right to the dollar (or, for eligibility flags, to the boolean). This is the deployability bar. A tax filer, a benefit estimator, or a caseworker cannot ship “close” — a prediction off by $50 is no more usable than one off by $500, because neither matches the bottom line a downstream system or a household needs. For currency amounts, “exact” means within 1 currency unit of the reference value after numeric parsing. For binary outputs, “exact” means the parsed value is exactly the same 0/1 flag as the reference.
Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 86% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 86% unweighted, leaving little room above the floor to separate models that actually understand the rules.
-
The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 86% row share to 71.1%. An always-zero baseline therefore scores 71.1% weighted exact rather than the 86% it earns unweighted, and the top model clears that baseline by 24.7 points — a margin comparable to within-1%. Across the 46-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.
+
The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 86% row share to 69.9%. An always-zero baseline therefore scores 69.9% weighted exact rather than the 86% it earns unweighted, and the top model clears that baseline by 25.6 points — a margin comparable to within-1%. Across the 47-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.
The exact rate is aggregated to a household score and then to a country score, with the same population-derived output weights that the within-1% companion and the bounded score use. Equal household weight preserves comparability across scenarios and slices. The leaderboard reports US results on this fixed test set.
Near-miss companion: within 1%
The within-1% hit rate is the near-miss-tolerant companion to the headline: the share of requested outputs the model gets within one percent of the PolicyEngine reference value. For currency amounts, “within 1%” means |pred − ref| ≤ 0.01 × |ref| when the reference is nonzero and |pred| ≤ 1 (the same one-currency-unit absolute tolerance used by exact match) when the reference is zero. Binary outputs are requested as integer 0/1 eligibility flags and scored identically to the exact-match metric.
@@ -179,8 +179,8 @@
Secondary me
Aggregation proceeds in three steps. First, each household-output prediction receives a 0-100 score. Second, each household receives one model score: requested output rows are weighted by output-group weights constructed from the full weighting population, then renormalized within the household over outputs that are actually requested for that household. Person-level coverage flags are scored at the person row; their output-group weight is split across the relevant people in the household and is based on PolicyEngine value proxies where available. Third, the country score averages those household scores with equal household weight. The leaderboard reports US results on this fixed test set. Equal-output-group scores and other alternative views are reported as sensitivity checks.
The benchmark requires each model response to include numeric answers and one explanation per requested output. Both the exact-match rate and the bounded score use only the numeric answers. Explanations are retained for scenario exploration and qualitative error analysis; they should not be interpreted as faithful traces of model reasoning.
Because explanations are required, the canonical task measures policy estimation under a public-facing structured-response contract, not isolated arithmetic accuracy. Prompt fairness is part of the benchmark contract. The current release uses one prompt template per country, with no model-specific tuning. Models receive the same household facts and requested outputs. They receive no web or external tool access. The prompt sets unlisted numeric inputs to 0, unlisted boolean or status facts to false, and household characteristics as constant over the tax-benefit year. Provider-specific differences are limited to structured-output transport, request shape, and the reasoning setup in Table 2. The harness sends no reasoning-control or sampling parameters. It leaves reasoning effort, temperature, and related decoding controls at provider defaults. The frozen GPT-5.5 wave uses provider-default reasoning effort.
-
The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 1.0 points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.
-
Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 46 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.
+
The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 0.8 points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.
+
Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 47 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.
One row, GLM-5.3-Flash (preview), answered as a cloaked OpenRouter listing called “Ox Alpha” and is labeled a preview. It was publicly callable under the canonical whole-scenario request and forced-tool transport, and its date is its OpenRouter listing date. After the row’s run, Z.ai identified the listing as GLM-5.3-Flash (OpenRouter 2026); the row carries the disclosed name and keeps the preview qualifier, because whether the checkpoint that answered is the one later released under that name has not been stated, and it is distinct from the GLM-5.3 row, which is Z.ai’s larger model called under its own id. Its artifact key stays ox-alpha, the name it was queried under.
Frozen snapshot and open-set status
Leaderboard positions are version-sensitive, so manuscript claims refer to the frozen source-run exports in Table 1 rather than to the live site. The committed source-run exports are the manuscript artifacts. The public site exposes the current prompts, predictions, explanations, and reference outputs for transparency. This makes the public leaderboard open-set: models, model providers, or benchmark users could learn from the released cases before later runs. Protected leaderboard claims would require a separate held-out or rotating set.
@@ -206,11 +206,11 @@
Frozen snapshot an
Snapshot date
-
2026-09-30
+
2026-10-10
Model response date
-
2026-06-12 to 2026-09-29
+
2026-06-12 to 2026-10-08
Policy period
@@ -222,7 +222,7 @@
Frozen snapshot an
Frozen export SHA-256 prefix
-
US b1da3eae058c
+
US 535a51db14f1
Snapshot manifest
@@ -238,7 +238,7 @@
Frozen snapshot an
US reference outputs SHA-256 prefix
-
e8bbba8fd3e9
+
fdd17a57092d
Benchmark spec SHA-256 prefix
@@ -262,7 +262,7 @@
Frozen snapshot an
Reference engine
-
policyengine-us 2.15.17
+
policyengine-us 2.38.6
PolicyEngine.py (recorded for provenance)
@@ -278,7 +278,7 @@
Frozen snapshot an
Models
-
46
+
47
Output groups
@@ -299,12 +299,12 @@
Frozen snapshot an
-
+
-Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through its row’s transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for 17 rows; tool choice for 16 rows; reasoning setup and timeouts for seven rows. Reasoning setup and timeouts for the other ten fingerprinted rows, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date.
+Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through its row’s transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for 18 rows; tool choice for 17 rows; reasoning setup and timeouts for eight rows. Reasoning setup and timeouts for the other ten fingerprinted rows, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date.
@@ -333,7 +333,7 @@
Frozen snapshot an
Claude Fable 5
claude-fable-5
2026-06-09
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
thinking does not engage under forced tool call
@@ -342,7 +342,7 @@
Frozen snapshot an
Claude Fable 5.1
claude-fable-5-1
2026-09-01
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 16,384-token shared budget
@@ -351,394 +351,403 @@
Frozen snapshot an
Claude Haiku 4.5
claude-haiku-4-5-20251001
2025-10-15
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
provider default
+
Claude Haiku 5.5
+
claude-haiku-5-5
+
2026-10-07
+
1,926/1,926
+
forced tool
+
whole scenario
+
provider default; 16,384-token shared budget
+
+
Claude Opus 4.7
claude-opus-4-7
2026-04-16
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
provider default
-
+
Claude Opus 4.8
claude-opus-4-8
2026-05-28
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
provider default
-
+
Claude Opus 5
claude-opus-5
2026-07-24
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
thinking does not engage under forced tool call
-
+
Claude Opus 5.5
claude-opus-5-5
2026-09-21
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 16,384-token shared budget
-
+
Claude Sonnet 4.6
claude-sonnet-4-6
2026-02-17
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
provider default
-
+
Claude Sonnet 5
claude-sonnet-5
2026-06-30
-
1,920/1,920
+
1,926/1,926
forced tool
1 output/request
thinking does not engage under forced tool call
-
+
Claude Sonnet 5.5
claude-sonnet-5-5
2026-09-28
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 16,384-token shared budget
-
+
DeepSeek V4 Flash 0731
openrouter/deepseek/deepseek-v4-flash-0731
2026-07-31†
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 98,304-token shared budget
-
+
DeepSeek V4 Pro
deepseek/deepseek-v4-pro
2026-04-24†
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 16,384-token shared budget
-
+
DeepSeek V4 Pro 0813
openrouter/deepseek/deepseek-v4-pro-0813
2026-08-13†
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 98,304-token shared budget
-
+
DeepSeek V4.1 Flash
deepseek/deepseek-flash
2026-09-10†
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 16,384-token shared budget
-
+
Gemini 3 Flash Preview
gemini/gemini-3-flash-preview
2025-12-17
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default
-
+
Gemini 3.1 Flash Lite Preview
gemini/gemini-3.1-flash-lite-preview
2026-03-03
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default
-
+
Gemini 3.1 Pro Preview
gemini/gemini-3.1-pro-preview
2026-02-19
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default
-
+
Gemini 3.5 Flash
gemini/gemini-3.5-flash
2026-05-19
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default
-
+
Gemini 3.5 Flash-Lite
gemini/gemini-3.5-flash-lite
2026-07-21
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Gemini 3.6 Flash
gemini/gemini-3.6-flash
2026-07-21
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Gemini 3.7 Flash
gemini/gemini-3.7-flash
2026-08-13
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Gemini 3.8 Flash
gemini/gemini-3.8-flash
2026-09-02
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GLM-5.2
openrouter/z-ai/glm-5.2
2026-06-13†
-
1,788/1,920
+
1,790/1,926
JSON
3 outputs/request
provider default; 16,384-token shared budget
-
+
GLM-5.3
openrouter/z-ai/glm-5.3
2026-08-14†
-
1,849/1,920
+
1,853/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-5.4 mini
gpt-5.4-mini
2026-03-17
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default
-
+
GPT-5.4 nano
gpt-5.4-nano
2026-03-17
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default
-
+
GPT-5.5
gpt-5.5
2026-04-23
-
1,920/1,920
+
1,926/1,926
forced tool
3 outputs/request
provider default; 16,384-token shared budget
-
+
GPT-5.6 Luna
gpt-5.6-luna
2026-07-09
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-5.6 Sol
gpt-5.6-sol
2026-07-09
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-5.6 Terra
gpt-5.6-terra
2026-07-09
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-6 Astra
gpt-6-astra
2026-09-04
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-6 Luna
gpt-6-luna
2026-09-22
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-6 Sol
gpt-6-sol
2026-09-22
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GPT-6.1 Sol
gpt-6.1-sol
2026-09-29
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Grok 4.3
xai/grok-4.3
2026-04-17
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default
-
+
Grok 4.5
xai/grok-4.5
2026-07-08
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Grok 4.6
xai/grok-4.6
2026-08-12
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Grok 4.7
xai/grok-4.7
2026-09-21
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Grok Build 0.1
xai/grok-build-0.1
2026-05-29
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default
-
+
Inkling
openrouter/thinkingmachines/inkling
2026-07-15†
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 49,152-token shared budget
-
+
Kimi K2.6
openrouter/moonshotai/kimi-k2.6
2026-04-20†
-
1,537/1,920
+
1,536/1,926
JSON
3 outputs/request
provider default; 16,384-token shared budget
-
+
Kimi K3
openrouter/moonshotai/kimi-k3
2026-07-16†
-
1,862/1,920
+
1,866/1,926
JSON
whole scenario
provider default; 49,152-token shared budget
-
+
MiniMax M3
openrouter/minimax/minimax-m3
2026-06-01†
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
GLM-5.3-Flash (preview)
openrouter/stealth/ox-alpha
2026-08-20
-
1,920/1,920
+
1,926/1,926
forced tool
whole scenario
provider default; 16,384-token shared budget
-
+
Qwen3.7-max
openrouter/qwen/qwen3.7-max
2026-05-19
-
1,920/1,920
+
1,926/1,926
JSON
3 outputs/request
provider default; 16,384-token shared budget
-
+
Qwen3.8-Max
openrouter/qwen/qwen3.8-max
2026-08-03
-
1,920/1,920
+
1,926/1,926
JSON
whole scenario
provider default; 98,304-token shared budget
@@ -802,14 +811,16 @@
Disability in the household facts
Reference-output credibility
PolicyBench treats PolicyEngine outputs as benchmark reference outputs, not as administrative records. The reference source is nevertheless stronger than an ad hoc answer key: PolicyEngine is open source, used for household calculators and reform analysis, and externally checked in specific domains. No. 10 Downing Street’s data science team adapted PolicyEngine’s open-source microsimulation model for experimental policy simulation, with validation against external projections before use (Woodruff 2026; Ghenis 2026). In the US, PolicyEngine (2024) reports matching the National Bureau of Economic Research (NBER) TAXSIM-35 model (Feenberg and Coutts 1993) to the cent on the vast majority of cases for the 2021 tax year across hundreds of thousands of tax units per state, with state-specific differences documented in the integration tests. We do not restate that comparison as a single percentage because the published source uses qualitative phrasing rather than a headline accuracy number. PolicyEngine has also signed a memorandum of understanding (Ghenis and Makarchuk 2025) with the Federal Reserve Bank of Atlanta for future validation work against its Policy Rules Database (2026). The Atlanta Fed sources are a caveat rather than evidence of completed validation for this benchmark: they document planned collaboration and the comparison source, not finished checks of the frozen PolicyBench outputs. Taken together, these sources support using PolicyEngine as a transparent reference implementation with partial external validation, but they do not validate every benchmark output.
-
This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered 7,527 rows whose legacy threshold score is below 1. That selected universe contains 7,523 exact-match misses and 4 exact hits; 2,072 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s scored reference outputs come from the fixed engine, policyengine-us 2.15.17. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. Table 4 summarizes the reviewed discrepancy classes and outcomes.
-
The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged 39 references. Adjudication against primary sources and sandbox runs of the reference engine affirmed 5 of them, traced 8 to inputs the prompt never lists, replaced 6 with a regenerated reference, and confirmed 20 as defects in policyengine-us 1.755.4, the version behind the references then. The judge’s stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause’s rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, 8 had been fixed in policyengine-us after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and #9586): capital gain distributions, New York’s renter cap, the CalEITC’s lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, 15 in all, and policyengine-us 2.15.17 contains all 8 fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin’s 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin’s published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model: 28 outputs in 20 households are recorded as engine-defect exclusions across 11 root causes, and 8 of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). They stay excluded until references built on a fixed engine replace them. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household’s survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.
+
This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered 8,001 rows whose legacy threshold score is below 1. That selected universe contains 7,997 exact-match misses and 4 exact hits; 2,138 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s scored reference outputs come from the fixed engine, policyengine-us 2.38.6. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. Table 4 summarizes the reviewed discrepancy classes and outcomes.
+
The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged 30 references. Adjudication against primary sources and sandbox runs of the reference engine affirmed 5 of them, traced 8 to inputs the prompt never lists, replaced 5 with a regenerated reference, and confirmed 11 as defects in policyengine-us 1.755.4, the version behind the references then. The judge’s stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause’s rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, 8 had been fixed in policyengine-us after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and #9586): capital gain distributions, New York’s renter cap, the CalEITC’s lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, 15 in all, and policyengine-us 2.38.6 contains all 8 fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin’s 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin’s published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model. The exclusion record holds 14 engine-defect exclusions in 12 households across 8 root causes, and 3 of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). policyengine-us 2.38.6 still has the defect behind 9 of them, across 5 root causes. An output returns to scoring when references built on an engine that fixes its defect replace it, unless it also moves under an input the prompt does not state or its reference rests on a further defect that engine has. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household’s survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.
The same wave tested a failure mode no judge flagged. policyengine-us fills a parameter’s values past its last encoded year by projecting it with a price index, so a 2026 reference can rest on a forecast rather than on the amount a government published. Every reference was recomputed with each projected 2026 value held at its last encoded value, and each projected amount behind a moved output was then sourced. Where the government published the 2026 amount before the freeze, the reference takes it: Minnesota’s, Michigan’s and Missouri’s brackets, deductions and exemptions, and Maryland’s withholding allowance. policyengine-us reads the allowance only in its estimate of Maryland income tax withheld, which fills the state income tax part of the federal state and local tax deduction; the one Maryland federal output that estimate moves is excluded (reference_audit/2026-10-05). Wisconsin had published its 2026 brackets and deductions too, but every Wisconsin output they move is excluded for an engine defect, so they enter only those outputs’ corrected values. Where no 2026 amount published before the freeze was found, the reference takes the last amount published: California’s indexed amounts, whose 2026 factor rests on June 2026 prices; the IRS optional sales tax tables, whose 2026 edition had not been found by September 22; Idaho’s zero-rate threshold; Maryland’s return deduction; and SNAP’s FY2027 figures, published on 2026-08-21. Illinois’s exemption and Minnesota’s child credit projections matched the published amounts. These corrections change parameter values and no rule, so the affected references were regenerated with the same engine version rather than excluded: 23 references, most of them SNAP. With the upstream fixes, the September 22 release regenerated 26 references in 24 households, 13 of them SNAP. The reference sidecar records each regenerated value with the fix that produced it, and the fixes, the recomputation of every reference under each, and the verification reports are committed with the snapshot (reference_audit/2026-09-22).
The SNAP convention as first built also corrected the engine’s SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine’s household size, eligibility and floored net income, the Axiom rules engine’s encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey’s state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income’s cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect is not fixed upstream: the engine still grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, and the scored output it moves is excluded. That household’s prompt calls its head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits; read the other way, the allowance would stand, and so would the reference the publication conventions and upstream SNAP fixes give ($3,576, which the output would carry if scored), so the output is excluded on either reading. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine’s values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA’s SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan’s broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0.
PolicyBench rebuilt the references on 2026-09-29 with policyengine-us 2.15.17, the newest release when it began sweeping the references that day (uploaded 00:23 UTC), in place of 1.755.4. It computes each scored reference with policyengine_us.Simulation. The snapshot also records policyengine.py 6.1.2 for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside 2.15.17. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA’s FY2027 figures as published values. policyengine-us 2.15.17 counts Maryland county income tax in its state income tax. PolicyBench’s state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP’s work rules read, so the scenario builder also passes each prompt’s stated usual weekly hours under that input’s name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes 4 scored references (Table 4). Two take law published before the freeze that policyengine-us 1.755.4 did not encode: New Jersey’s child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona’s broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York’s Empire State child credit phase-out. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household’s excluded SNAP output also flagged a scored output the move did not change: the head’s Medicaid eligibility. The head’s income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California’s Working Disabled Program requires SSI’s definition of disability, the unlisted input that already excludes the household’s SNAP. policyengine-us 2.15.17 tests the general disability flag there and gives 1 under either reading; with the program’s test reading SSI’s definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another 2 references move by less than the dollar tolerance. PolicyBench re-reviewed the 19 excluded outputs whose values move on 2.15.17; all stay excluded, each with the value it was decided on. policyengine-us 2.17.0, the newest release when PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as 2.15.17 for all 1,984 outputs under the same conventions and adapter. The reference sidecar lists every change, and reference_audit/2026-09-28 holds the recomputation of every output, the investigations and the reviews.
-
A review of release dashboard-data-20260930 on 2026-10-05 found three more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with state_withheld_income_tax, a formula estimate of withholding on the person’s whole federal adjusted gross income rather than the household’s state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of five households moves, three of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt’s instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of one household move: it itemizes, so its federal income tax moves, and Virginia’s itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer’s choice. The four payroll references from those states count the cap, and the prompt does not state the employer’s choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California’s, Connecticut’s, New Jersey’s and Pennsylvania’s employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us 2.15.17 with the publication conventions, under each reading, and each sweep’s baseline reproduces all 1,928 references that release scored. Beyond the one-dollar tolerance, the sweeps move eight scored outputs, which PolicyBench excluded, and two federal income tax outputs already excluded for other reasons; they move no output that is still scored by any amount. The eight records are computed on policyengine-us 2.15.17, whose values are the references. The 64 excluded outputs now keep the values they were decided on (52 computed with policyengine-us 1.755.4, 12 with 2.15.17).
+
A review of release dashboard-data-20260930 on 2026-10-05 found three more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with state_withheld_income_tax, a formula estimate of withholding on the person’s whole federal adjusted gross income rather than the household’s state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of five households moves, three of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt’s instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of one household move: it itemizes, so its federal income tax moves, and Virginia’s itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer’s choice. The four payroll references from those states count the cap, and the prompt does not state the employer’s choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California’s, Connecticut’s, New Jersey’s and Pennsylvania’s employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us 2.15.17 with the publication conventions, under each reading, and each sweep’s baseline reproduces all 1,928 references that release scored. Beyond the one-dollar tolerance, the sweeps move eight scored outputs, which PolicyBench excluded, and two federal income tax outputs already excluded for other reasons; they move no output that is still scored by any amount. The eight records are computed on policyengine-us 2.15.17, whose values are the references.
+
On 2026-10-06 PolicyBench ruled to exclude ten more outputs. Eight came from a reference adversary (reference_audit/2026-10-05-reference-adversary). It flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give. A judge that cannot yet see how the engine derived the reference works each flagged output from primary law, and only then reconciles its answer with the derivation. The adversary flagged 61 outputs and ran 52, the other nine belonging to other audits, and it held 45 references. An independent check of its seven other verdicts confirmed four as engine defects: Arizona’s 2026 standard deduction left unindexed (A.R.S. 43-1041(H)), Ohio’s medical deduction counting health insurance premiums only for a Medicare-eligible filer with no employer-paid plan (R.C. 5747.01(A)(10)), Colorado’s 2026 sales tax refund paid without the excess state revenues that C.R.S. 39-22-2003(2) requires, and New York’s 2026 child and dependent care credit, which Tax Law 606(c-2) replaced. The check refuted two, the Medicaid eligibility of two children in a North Carolina household, whose references stand. The seventh is a question of definition. A Pennsylvania household’s dependent earns $45,000 of wages and must file their own return, and the output definitions name no return, so the household’s federal and state income tax can be read with or without that return; a scan for the same question found a Missouri household, and PolicyBench excluded the four outputs. The other two outputs are two Louisiana households’ 2026 state income tax (reference_audit/2026-10-05-louisiana). Their references rest on a standard deduction that policyengine-us computed from published price indexes under Louisiana’s formula; Louisiana published the return amount itself on 2026-09-28, after the freeze. All ten records are computed on policyengine-us 2.15.17.
+
On 2026-10-10 PolicyBench moved the references again, from policyengine-us 2.15.17 to 2.38.6, a release that fixes engine defects behind excluded outputs. With the same publication conventions and adapter, it recomputed every output (reference_audit/2026-10-09-engine-upgrade). policyengine-us 2.38.6 was the newest release when PolicyBench began sweeping the references on 2026-10-10 (uploaded 05:39 UTC). It was still the newest release when PolicyBench checked PyPI on 2026-10-10 at 06:53 UTC. The move fixes the engine defects behind 18 outputs. Earlier releases excluded 14 of them, which return to scoring at the new version’s values; the other four, ruled on 2026-10-06 while still scored, stay scored at those values. Of the 18, 16 land within $1 of the corrected values their exclusion records carry. The other two are held instead to the values the audit’s fix modules give on policyengine-us 2.37.2, an engine that still has the defects; each lands within $1 of its own. The reference sidecar records each target with the upstream fix it names (regenerated_exclusions). The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The move changes one scored references beyond the dollar tolerance and 2 within it, and removes two outputs from scoring. The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax). PolicyBench re-reviewed the 17 excluded outputs whose values move on 2.38.6; all stay excluded, each with the value it was decided on. The 58 excluded outputs now keep the values they were decided on (38 computed with policyengine-us 1.755.4, 18 with 2.15.17, 2 with 2.38.6).
The audit tooling itself required hardening, which we report because it changes how much the annotations can be trusted. The derivation narratives displayed alongside reference values are generated from engine computation traces. After earlier model-written narratives were found to misattribute eligibility pathways, mechanical validators now enforce that each narrative states the engine’s actual mechanism — for example, that a child ineligible for CHIP because they qualify for Medicaid is never described as failing a CHIP income test. The failure-audit judge reads these grounded narratives together with engine facts and must return a per-model diagnosis naming the specific rule or computation the model missed. It is barred from re-adjudicating the reference in prose: hedged or reference-adjudicating verdicts are mechanically rejected and re-judged, and genuine reference doubt is routed to a structured flag that triggers the manual adjudication described above.
-
After freezing the snapshot and completing response-contract repairs, we annotated the 7,527 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 7,523 of the snapshot’s 7,523 exact-match misses and 4 exact hits. The 2,072 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). Two further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. prompt_ambiguity marks 36 outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine’s Working Disabled Program test read SSI’s definition, as the program does). reference_engine_defect marks the 28 outputs where that engine misapplies the law on stated facts. Together they remove 64 outputs in 42 households, and of the 2,444 annotated rows on those outputs, 2,381 carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP’s October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on 1,920 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 77 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the 116 cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29 and September 30 additions joined or a reference revision changed (260 of 674 cases), its own row’s cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build; months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (weekly_hours_worked_before_lsr), which policyengine-us 1.755.4 defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin’s homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers’ child, which decides whose income counts in that dependent’s Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general is disabled fact and never a program-specific determination (Section 4.2). Every exclusion but the California Medicaid output was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect’s sandbox fix, moved on policyengine-us 1.755.4 or, for the state and local tax refund reading and the readings of the 2026-10-05 review, on 2.15.17. On 1.755.4, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On 2.15.17, PolicyBench re-ran four of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves three previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same 2.15.17 calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and three scored outputs move by less. PolicyBench did not re-run the other sweeps on 2.15.17. On 2.15.17, the California Medicaid output stays at 1 under either reading of SSI’s disability criterion, because the engine’s Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household’s excluded SNAP output, and excluded it because the program itself requires SSI’s definition of disability.
+
After freezing the snapshot and completing response-contract repairs, we annotated the 8,001 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 7,997 of the snapshot’s 7,997 exact-match misses and 4 exact hits. The 2,138 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). Three further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. prompt_ambiguity marks 42 outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine’s Working Disabled Program test read SSI’s definition, as the program does). reference_engine_defect marks the 14 outputs where that engine misapplies the law on stated facts. policyengine-us 2.38.6 computes four of them at their corrected values; they stay excluded because each also moves under an input the prompt does not state. policyengine-us 2.38.6 computes one more at its recorded corrected value, but its reference rests on a further engine defect, found on 2.38.6 and not fixed there, so it stays excluded. reference_later_law marks the two Louisiana outputs whose reference rests on an amount the state published after the freeze. Together they remove 58 outputs in 42 households, and of the 2,212 annotated rows on those outputs, 2,164 carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP’s October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on 1,926 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 70 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the 94 cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29, September 30 and October 9 additions joined or a reference revision changed (314 of 677 cases), its own row’s cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build; months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (weekly_hours_worked_before_lsr), which policyengine-us 1.755.4 defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin’s homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers’ child, which decides whose income counts in that dependent’s Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general is disabled fact and never a program-specific determination (Section 4.2). Every exclusion but the California Medicaid output and the 2026-10-06 records was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect’s sandbox fix, moved on policyengine-us 1.755.4 or, for the state and local tax refund reading and the readings of the 2026-10-05 review, on 2.15.17; the outputs the 2026-10-10 upgrade excluded are ones policyengine-us 2.38.6 itself moved, onto an input no prompt states. The 2026-10-06 records are the exception by design. The reference adversary selects the outputs where models agree against the reference, and an independent check against primary law then decided each; the two Louisiana records came from a sweep of that state’s 2026 standard deduction. On 1.755.4, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On 2.15.17, PolicyBench re-ran four of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves three previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same 2.15.17 calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and three scored outputs move by less. PolicyBench did not re-run the other sweeps on 2.15.17. On 2.15.17, the California Medicaid output stays at 1 under either reading of SSI’s disability criterion, because the engine’s Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household’s excluded SNAP output, and excluded it because the program itself requires SSI’s definition of disability.
@@ -879,6 +890,10 @@
Reference-output credibility
Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us 1.755.4 and 2.15.17 to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes 4 scored references, for law published before the freeze that 1.755.4 did not encode or misapplied: New Jersey’s 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona’s 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York’s Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: 2.15.17 counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the 19 excluded outputs that move; all stay excluded. The re-review of a California household’s excluded SNAP output flagged its head’s Medicaid eligibility, which turns on the same unlisted input, SSI’s disability criterion, through the Working Disabled Program; PolicyBench excluded it.
+
US references on policyengine-us 2.38.6 (October 2026 upgrade)
+
The move from policyengine-us 2.15.17 fixes the engine defects behind 18 outputs (14 that earlier releases excluded return to scoring at the new version’s values; the 4 ruled on 2026-10-06 while still scored stay scored) and changes 1 scored reference beyond the dollar tolerance; 2 outputs leave scoring. PolicyBench re-reviewed the 17 excluded outputs whose values move; all stay excluded. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).
+
+
US federal and Virginia income tax (October 2026 review)
No judge flag: a review of release dashboard-data-20260930 swept every reference on policyengine-us 2.15.17 under readings of two inputs no prompt states. The state income tax part of the federal state and local tax deduction is policyengine-us’s formula estimate of state income tax withheld on federal adjusted gross income; read as the household paying its 2026 liability during the year, or paying none, three scored federal income tax outputs move, and PolicyBench excluded them; the readings also move two federal outputs already excluded. policyengine-us treats every Medicare-eligible person as enrolled and counts a modeled Part B premium as a medical expense; without it, one household’s federal and Virginia income tax move, and both are excluded.
@@ -921,18 +936,18 @@
Reference-output credibility
US
-
7527
-
6883
-
644
+
8001
+
7342
+
659
0
0
0
Total
-
7527
-
6883
-
644
+
8001
+
7342
+
659
0
0
0
@@ -946,7 +961,7 @@
Reference-output credibility
Results
United States leaderboard
-
The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-6 Sol (95.8 exact, 95% CI 94.2–97.3; 96.5 within-1%), Claude Opus 5.5 (95.1 exact, 95% CI 93.2–96.7; 96.0 within-1%), and GPT-5.6 Sol (94.4 exact, 95% CI 92.2–96.3; 94.9 within-1%).
+
The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-6 Sol (95.6 exact, 95% CI 93.9–97.0; 96.2 within-1%), Claude Opus 5.5 (94.4 exact, 95% CI 92.5–96.1; 95.8 within-1%), and GPT-5.6 Sol (93.5 exact, 95% CI 91.0–95.6; 94.4 within-1%).
@@ -987,51 +1002,51 @@
United States leaderboard
0
GPT-6 Sol
-
95.8
-
94.2–97.3
+
95.6
+
93.9–97.0
1–2
-
96.5
-
97.3
-
98.1
+
96.2
+
97.6
+
98.2
1
Claude Opus 5.5
-
95.1
-
93.2–96.7
+
94.4
+
92.5–96.1
1–3
-
96.0
-
97.1
-
97.7
+
95.8
+
97.3
+
98.0
2
GPT-5.6 Sol
+
93.5
+
91.0–95.6
+
2–7
94.4
-
92.2–96.3
-
1–6
-
94.9
-
96.3
-
97.0
+
96.1
+
97.2
3
Claude Sonnet 5.5
-
93.3
-
91.0–95.3
-
3–7
-
94.1
-
96.3
-
97.1
+
92.2
+
89.8–94.3
+
3–8
+
93.4
+
96.0
+
96.9
4
GPT-6 Luna
-
93.0
-
90.9–94.9
+
92.0
+
89.8–94.0
3–9
-
94.5
-
97.1
+
93.9
+
96.8
97.5
@@ -1045,9 +1060,9 @@
United States leaderboard
Uncertainty
Each leaderboard score is a mean over a sample of 100 households, so the rankings carry sampling uncertainty. We quantify it with a household-level bootstrap: the 100 households are resampled with replacement 10,000 times, and each model’s population-weighted exact-match rate is recomputed on every resample. The resampling unit is the household, carried with its full output vector, so within-household correlations across outputs are preserved. The same resampled set is used for every model in each replicate (a paired bootstrap), so the rank ranges in Table 6 reflect genuine ordering uncertainty rather than independent per-model noise.
-
Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 7 of the other 45 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.
+
Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 6 of the other 46 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.
Cost and latency
-
Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.002 (GPT-5.4 nano) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.8% exact, sits mid-pack at $0.027 per household and 23 s.
+
Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.6% exact, sits mid-pack at $0.027 per household and 23 s.
@@ -1068,277 +1083,283 @@
Cost and latency
GPT-6 Sol
-
95.8
+
95.6
$0.027
23 s
Claude Opus 5.5
-
95.1
+
94.4
$0.067
24 s
GPT-5.6 Sol
-
94.4
+
93.5
$0.090
36 s
Claude Sonnet 5.5
-
93.3
+
92.2
$0.034
20 s
GPT-6 Luna
-
93.0
+
92.0
$0.002
29 s
GPT-6 Astra
-
92.5
+
91.9
$0.126
32 s
Claude Fable 5.1
-
92.1
+
91.3
$0.257
49 s
-
GPT-6.1 Sol
-
91.3
-
$0.023
-
34 s
-
-
Kimi K3
-
91.2
+
91.0
$0.470
1023 s
+
+
GPT-6.1 Sol
+
90.3
+
$0.023
+
34 s
+
GPT-5.6 Luna
-
90.3
+
89.2
$0.019
14 s
GLM-5.3-Flash (preview)
-
89.8
+
89.0
—
242 s
+
GPT-5.5
+
88.2
+
$0.262
+
—
+
+
Inkling
-
89.3
+
88.2
$0.091
250 s
-
+
Grok 4.7
-
89.3
+
88.1
$0.260
547 s
-
-
GPT-5.5
-
89.1
-
$0.262
-
—
-
GPT-5.6 Terra
-
89.0
+
88.0
$0.039
19 s
DeepSeek V4.1 Flash
-
88.0
+
86.8
$0.024
85 s
Grok 4.6
-
87.5
+
86.2
$0.087
219 s
Gemini 3.8 Flash
-
87.2
+
86.0
$0.068
56 s
Grok 4.5
-
86.3
+
85.1
$0.073
112 s
Claude Opus 5
-
85.2
+
84.0
$0.067
16 s
Claude Fable 5
-
84.9
+
83.7
$0.541
98 s
DeepSeek V4 Pro 0813
-
84.6
+
83.4
$0.199
808 s
DeepSeek V4 Flash 0731
-
84.5
+
83.3
$0.003
195 s
Gemini 3.7 Flash
-
84.2
+
83.1
$0.014
9 s
Gemini 3.6 Flash
-
84.2
+
83.0
$0.055
26 s
Gemini 3.5 Flash-Lite
-
83.2
+
82.0
$0.003
3 s
Gemini 3.1 Pro Preview
-
82.8
+
81.6
$0.085
45 s
GLM-5.3
-
82.3
+
81.4
$0.169
134 s
Gemini 3 Flash Preview
-
82.2
+
81.0
$0.048
58 s
Claude Opus 4.7
-
82.2
+
80.9
$0.293
53 s
Grok 4.3
-
81.9
+
80.7
$0.015
22 s
-
Grok Build 0.1
-
81.6
-
$0.045
-
135 s
-
-
Claude Sonnet 4.6
-
81.5
+
80.3
$0.186
114 s
+
+
Grok Build 0.1
+
80.2
+
$0.045
+
135 s
+
Gemini 3.5 Flash
-
81.3
+
80.1
$0.056
24 s
Gemini 3.1 Flash Lite Preview
-
80.9
+
79.7
$0.002
3 s
DeepSeek V4 Pro
-
80.8
+
79.7
$0.011
170 s
+
Claude Haiku 5.5
+
79.6
+
$0.001
+
5 s
+
+
Qwen3.7-max
-
78.6
+
77.4
$0.139
—
-
+
GLM-5.2
-
78.0
+
76.9
$0.082
—
-
+
Claude Opus 4.8
-
77.4
+
76.2
$0.255
55 s
-
+
MiniMax M3
-
76.9
+
75.8
$0.007
—
-
+
Qwen3.8-Max
-
76.4
+
75.2
$0.224
861 s
-
+
Claude Haiku 4.5
-
75.9
+
74.8
$0.055
48 s
-
+
GPT-5.4 mini
-
74.7
+
73.6
$0.006
5 s
-
+
Claude Sonnet 5
-
73.8
+
72.7
$0.172
64 s
-
+
Kimi K2.6
-
68.1
+
67.2
$0.391
—
-
+
GPT-5.4 nano
-
66.5
+
65.6
$0.002
6 s
@@ -1351,7 +1372,7 @@
Cost and latency
Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5’s usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.
Simple baselines
-
Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 71.1% on the headline exact-match rate — below the 86% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 24.7 points, and the within-1% companion shows the same separation with near-miss tolerance.
+
Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 69.9% on the headline exact-match rate — below the 86% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.6 points, and the within-1% companion shows the same separation with near-miss tolerance.
@@ -1389,16 +1410,16 @@
Simple baselines
0
Always zero
-
71.1
-
71.1
-
71.1
+
69.9
+
69.9
+
69.9
1
Median reference by output
-
65.1
-
65.1
-
67.5
+
63.9
+
63.9
+
66.8
@@ -1448,44 +1469,44 @@
Sensitivity to benchmark view
0
Exact-match headline
-
GPT-6 Sol (95.8)
-
Claude Opus 5.5 (95.1)
-
GPT-5.6 Sol (94.4)
+
GPT-6 Sol (95.6)
+
Claude Opus 5.5 (94.4)
+
GPT-5.6 Sol (93.5)
1
Within-1% companion
-
GPT-6 Sol (96.5)
-
Claude Opus 5.5 (96.0)
-
GPT-5.6 Sol (94.9)
+
GPT-6 Sol (96.2)
+
Claude Opus 5.5 (95.8)
+
GPT-5.6 Sol (94.4)
2
Bounded score
-
GPT-6 Sol (98.1)
-
Claude Opus 5.5 (97.7)
+
GPT-6 Sol (98.2)
+
Claude Opus 5.5 (98.0)
GPT-6 Luna (97.5)
3
Equal-output-group score
-
Claude Opus 5.5 (98.2)
-
GPT-6 Sol (98.2)
+
Claude Opus 5.5 (98.3)
+
GPT-6 Sol (98.3)
GPT-6 Astra (98.1)
4
Amount outputs only
-
Claude Opus 5.5 (97.6)
-
GPT-6 Astra (97.6)
-
GPT-6.1 Sol (97.4)
+
Claude Opus 5.5 (98.0)
+
Claude Fable 5.1 (97.5)
+
GPT-6 Astra (97.5)
5
Positive reference cases only
-
GPT-6 Astra (94.9)
-
GPT-6.1 Sol (92.3)
-
Claude Opus 5.5 (91.6)
+
GPT-6 Astra (95.1)
+
GPT-6.1 Sol (92.7)
+
Claude Opus 5.5 (92.3)
6
@@ -1541,23 +1562,23 @@
Sensitivity to benchmark view
0
Household-weighted (default)
-
GPT-6 Sol (98.1)
-
Claude Opus 5.5 (97.7)
+
GPT-6 Sol (98.2)
+
Claude Opus 5.5 (98.0)
GPT-6 Luna (97.5)
1
Equal weights
-
Claude Opus 5.5 (98.3)
-
GPT-6 Sol (98.1)
-
GPT-6 Astra (97.9)
+
Claude Opus 5.5 (98.4)
+
GPT-6 Sol (98.3)
+
GPT-6 Astra (97.8)
2
Budget-weighted
-
GPT-6 Sol (97.7)
-
Claude Opus 5.5 (96.7)
-
GPT-6 Luna (96.7)
+
GPT-6 Sol (98.0)
+
Claude Opus 5.5 (97.3)
+
GPT-5.6 Sol (96.9)
@@ -1608,31 +1629,31 @@
Hardest benchmark targets
0
federal_income_tax_before_refundable_credits
-
82.9
-
64.0
-
65.7
-
73.1
+
82.6
+
60.6
+
62.9
+
71.4
15
state_income_tax_before_refundable_credits
84.4
-
65.1
-
67.4
+
64.0
+
66.3
76.5
16
state_refundable_credits
-
85.5
-
81.9
-
82.0
+
86.2
+
82.8
82.8
+
83.7
13
snap
-
89.5
+
89.4
84.2
85.5
88.0
@@ -1640,10 +1661,10 @@
Hardest benchmark targets
1
federal_refundable_credits
-
91.9
-
86.8
-
88.0
-
90.4
+
91.8
+
86.9
+
88.1
+
90.3
@@ -1693,58 +1714,58 @@
Zero and positive cases
0
federal_income_tax_before_refundable_credits
-
74.0
-
43.6
+
72.3
+
43.7
93.4
1
state_income_tax_before_refundable_credits
77.3
-
48.7
-
91.8
+
50.3
+
92.0
2
state_refundable_credits
-
83.9
-
17.2
-
98.8
+
84.7
+
17.9
+
98.6
3
snap
88.4
-
36.5
+
35.8
96.8
4
federal_refundable_credits
-
91.7
-
58.8
-
95.8
+
91.6
+
57.5
+
95.9
5
payroll_tax
-
92.1
-
88.2
+
91.8
+
87.7
98.5
6
person_medicaid_eligible
-
92.8
-
83.1
+
92.7
+
82.8
95.6
7
person_chip_eligible
-
96.4
+
96.5
—
-
96.4
+
96.5
@@ -1766,7 +1787,7 @@
Zero and positive cases
Open-weight time lag
-
The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 80.8% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 91.2% exact trails 8 closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).
+
The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 79.7% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 91.0% exact trails 7 closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).
Both figures are lower bounds in one direction: a closed model outside the evaluated roster that crossed either score earlier would lengthen the corresponding lag. They also carry the uncertainty of any single-benchmark, single-domain estimate — the AISI analysis reports substantially longer lags on cyber tasks — and a lag measured at a moving frontier compresses mechanically when a strong open-weight release lands days after a frontier release.
Failure modes
The benchmark surfaces a few recurring failure patterns.
@@ -1814,319 +1835,326 @@
Failure modes
93.8
-
33
+
34
GPT-6.1 Sol
99.0
-
92.8
-
92.8
+
93.8
+
93.8
-
6
+
7
Claude Opus 5.5
100.0
-
91.8
-
91.8
+
92.8
+
92.8
-
30
+
32
+
GPT-6 Luna
+
99.0
+
90.7
+
89.7
+
+
+
31
GPT-6 Astra
100.0
88.7
88.7
+
+
33
+
GPT-6 Sol
+
99.0
+
88.7
+
88.7
+
-
31
-
GPT-6 Luna
+
38
+
Grok 4.7
99.0
-
89.7
+
88.7
88.7
-
32
-
GPT-6 Sol
+
29
+
GPT-5.6 Sol
99.0
87.6
87.6
-
37
-
Grok 4.7
-
99.0
-
87.6
-
87.6
-
-
-
28
-
GPT-5.6 Sol
-
99.0
-
86.6
+
27
+
GPT-5.5
+
95.9
+
88.7
86.6
-
-
26
-
GPT-5.5
+
+
14
+
DeepSeek V4.1 Flash
95.9
87.6
85.6
-
-
21
+
+
22
Gemini 3.8 Flash
95.9
-
86.6
-
84.5
-
-
-
36
-
Grok 4.6
-
96.9
+
87.6
85.6
-
84.5
-
13
-
DeepSeek V4.1 Flash
-
95.9
+
37
+
Grok 4.6
+
96.9
+
86.6
85.6
-
83.5
-
12
+
13
DeepSeek V4 Pro 0813
94.8
-
84.5
-
82.5
+
85.6
+
83.5
-
19
+
20
Gemini 3.6 Flash
95.9
+
84.5
83.5
-
82.5
-
20
+
21
Gemini 3.7 Flash
91.8
-
85.6
-
81.4
+
86.6
+
82.5
-
39
+
40
Inkling
94.8
-
83.5
-
81.4
+
84.5
+
82.5
-
14
-
Gemini 3 Flash Preview
-
91.8
-
85.6
-
80.4
-
-
-
41
+
42
Kimi K3
94.8
-
82.5
-
80.4
+
83.5
+
81.4
-
+
0
Claude Fable 5
92.8
-
84.5
-
79.4
+
85.6
+
80.4
-
-
9
+
+
10
Claude Sonnet 5.5
99.0
+
81.4
80.4
-
79.4
-
-
11
+
+
12
DeepSeek V4 Pro
91.8
-
83.5
-
79.4
+
84.5
+
80.4
+
+
+
15
+
Gemini 3 Flash Preview
+
91.8
+
85.6
+
80.4
-
27
+
28
GPT-5.6 Luna
94.8
-
81.4
-
79.4
+
82.5
+
80.4
-
29
+
30
GPT-5.6 Terra
95.9
-
81.4
-
79.4
+
82.5
+
80.4
-
35
+
36
Grok 4.5
93.8
-
81.4
-
79.4
+
82.5
+
80.4
-
16
+
17
Gemini 3.1 Pro Preview
91.8
-
82.5
-
78.4
+
83.5
+
79.4
-
43
+
44
GLM-5.3-Flash (preview)
91.8
-
83.5
-
78.4
+
84.5
+
79.4
-
17
+
18
Gemini 3.5 Flash
91.8
-
81.4
-
77.3
+
82.5
+
78.4
-
5
-
Claude Opus 5
+
3
+
Claude Haiku 5.5
88.7
-
83.5
-
76.3
+
82.5
+
77.3
-
7
+
8
Claude Sonnet 4.6
93.8
-
81.4
-
76.3
+
82.5
+
77.3
-
10
+
6
+
Claude Opus 5
+
88.7
+
83.5
+
76.3
+
+
+
11
DeepSeek V4 Flash 0731
89.7
-
82.5
-
75.3
+
83.5
+
76.3
-
-
15
+
+
16
Gemini 3.1 Flash Lite Preview
86.6
-
81.4
-
75.3
+
82.5
+
76.3
-
-
18
+
+
19
Gemini 3.5 Flash-Lite
87.6
-
81.4
-
75.3
+
82.5
+
76.3
-
-
25
+
+
26
GPT-5.4 nano
87.6
-
81.4
-
75.3
+
82.5
+
76.3
-
-
38
+
+
39
Grok Build 0.1
90.7
81.4
75.3
-
-
4
+
+
5
Claude Opus 4.8
85.6
-
81.4
-
73.2
+
82.5
+
74.2
-
-
34
+
+
35
Grok 4.3
85.6
-
81.4
+
82.5
+
74.2
+
+
+
25
+
GPT-5.4 mini
+
82.5
+
82.5
73.2
-
23
+
24
GLM-5.3
87.6
79.4
72.2
-
24
-
GPT-5.4 mini
-
82.5
-
81.4
-
72.2
-
-
-
8
+
9
Claude Sonnet 5
81.4
81.4
71.1
-
-
42
+
+
43
MiniMax M3
81.4
-
81.4
-
70.1
+
82.5
+
71.1
-
-
3
+
+
4
Claude Opus 4.7
86.6
79.4
69.1
-
-
22
-
GLM-5.2
-
81.4
-
74.2
-
68.0
-
2
Claude Haiku 4.5
76.3
-
81.4
-
67.0
+
82.5
+
68.0
-
44
-
Qwen3.7-max
-
79.4
+
23
+
GLM-5.2
81.4
-
66.0
+
75.3
+
68.0
45
+
Qwen3.7-max
+
79.4
+
82.5
+
66.0
+
+
+
46
Qwen3.8-Max
68.0
-
81.4
-
58.8
+
82.5
+
59.8
-
-
40
+
+
41
Kimi K2.6
46.4
-
50.5
-
30.9
+
51.5
+
32.0
@@ -2137,19 +2165,19 @@
Failure modes
-
Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 644 cells (0.7%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.
+
Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 659 cells (0.7%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.
Limitations
PolicyBench is not a substitute for a production tax-and-benefit calculator. Several caveats matter:
output-contract reliability required a repair workflow; raw provider responses are retained within the canonical prediction files wherever the transport exposed them (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)), and cells that never parsed score zero
-
request shape is not identical across models: 10 of 46 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
+
request shape is not identical across models: 10 of 47 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
zero-heavy outputs require separate positive-case interpretation
the choice of headline metric (exact match), near-miss companion (within-1%), and secondary tracking metric (bounded score) is one benchmark view; all three are reported, but downstream evaluations should also consider error-magnitude metrics (mean absolute error, mean absolute percentage error) where appropriate
the public scenario explorer exposes the current test set and reference outputs, so open-set leakage is a prominent limitation rather than a minor implementation detail
the 100-household manuscript snapshot should be treated as a preview until larger frozen runs are published
-
the annotations cover the 7,527-row legacy-threshold audit universe, including 7,523 exact-match misses and 4 exact hits, but not 2,072 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
+
the annotations cover the 8,001-row legacy-threshold audit universe, including 7,997 exact-match misses and 4 exact hits, but not 2,138 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (disability, which the prompt states as one general fact where each program applies its own determination, and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address (Section 4.2); outputs whose reference moves under the other reading of either convention are excluded, while the mis-encoded scenario’s cells score models against its encoded inputs
-
the reference engine carries known defects: 28 outputs, recorded as engine-defect exclusions, rest on 11 root causes that the audit found in policyengine-us 1.755.4 and that 2.15.17 does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, the engine-upgrade review covered only the outputs the upgrade moved, and on 2.15.17 PolicyBench re-ran four of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
+
the reference engine carries known defects: 10 outputs, recorded as engine-defect exclusions, rest on 6 root causes that the audits found in policyengine-us 1.755.4 or later releases and that 2.38.6 does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, each engine-upgrade review covered only the outputs its upgrade moved, and on 2.15.17 PolicyBench re-ran four of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
the 2026-10-05 review of release dashboard-data-20260930 found eight scored references that rested on inputs the prompt never states and that no judge had flagged, so other such references can remain scored: the exclusions cover the inputs the audits found
the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP’s FY2026 figures rather than USDA’s FY2027 figures of 2026-08-21; California’s 2025 indexed amounts; the IRS’s 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey’s 2026 child tax credit schedule and Arizona’s SNAP categorical eligibility limit only after 1.755.4
benchmark success should not be interpreted as policy-advice readiness
@@ -2195,7 +2223,7 @@
Appendix
Final parse coverage
-
644 of the 88,320 canonical model-output rows (0.7%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (383), GLM-5.2 (132), GLM-5.3 (71), and Kimi K3 (58). They score zero as parse-contract failures. A further 60 rows (Grok 4.3 (55), Kimi K2.6 (4), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 704 rows in all fall short of the numeric-plus-explanation contract. Every other row has both.
+
659 of the 90,522 canonical model-output rows (0.7%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (390), GLM-5.2 (136), GLM-5.3 (73), and Kimi K3 (60). They score zero as parse-contract failures. A further 59 rows (Grok 4.3 (55), Kimi K2.6 (3), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 718 rows in all fall short of the numeric-plus-explanation contract. Every other row has both.
Preservation rule
@@ -2208,7 +2236,7 @@
Appendix
-
Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 644 cells (0.7% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (383), GLM-5.2 (132), GLM-5.3 (71), and Kimi K3 (58). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.
+
Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 659 cells (0.7% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (390), GLM-5.2 (136), GLM-5.3 (73), and Kimi K3 (60). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.
Competing interests
The author leads PolicyEngine, which develops the microsimulation software used to produce the benchmark reference outputs, and is a founder of the Axiom Foundation, a nonprofit developing deterministic rules-as-code infrastructure. Both organizations’ missions are advanced by the finding that unaided language models do not reliably reproduce rule calculations.
diff --git a/app/src/components/ExclusionNote.tsx b/app/src/components/ExclusionNote.tsx
index c372e97f..4ec0c1af 100644
--- a/app/src/components/ExclusionNote.tsx
+++ b/app/src/components/ExclusionNote.tsx
@@ -76,7 +76,7 @@ export default function ExclusionNote({
{formatValue(exclusion.frozenValue, isBinary, currencySymbol)}
{" "}
- used the engine’s projection.
+ used the engine’s own figure.
{exclusion.law ? `Law: ${exclusion.law}. ` : ""}
diff --git a/app/src/data.artifact.json b/app/src/data.artifact.json
index 640d7cc2..869eef55 100644
--- a/app/src/data.artifact.json
+++ b/app/src/data.artifact.json
@@ -1,9 +1,9 @@
{
"asset": "dashboard-data.json",
- "bytes": 127225036,
+ "bytes": 128756688,
"repo": "PolicyEngine/policybench",
- "sha256": "aa34e5c9ea926848dc7af460a98f17021bc42dae54a8015c9988a85e79a5d462",
- "tag": "dashboard-data-20261006",
- "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261006/dashboard-data.json",
+ "sha256": "f834478e65191f8426ef479506f9f6fe32c2e7107f30fdbf4e7ba0e6acba6410",
+ "tag": "dashboard-data-20261010",
+ "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261010/dashboard-data.json",
"version": 1
}
diff --git a/app/src/data.versions.json b/app/src/data.versions.json
index 95084590..3b6dd895 100644
--- a/app/src/data.versions.json
+++ b/app/src/data.versions.json
@@ -6,10 +6,10 @@
"artifact": {
"pointer": "live"
},
- "description": "Scored reference outputs from policyengine-us 2.15.17 (the 64 excluded outputs keep the values they were decided on: 52 from policyengine-us 1.755.4, 12 from 2.15.17, and the 19 that move on 2.15.17 were re-reviewed and stay excluded); same household facts and requested outputs for every model; answer transport and request shape per model as recorded in the serving-configuration table - 46 models",
+ "description": "Scored reference outputs from policyengine-us 2.38.6 (the 58 excluded outputs keep the values they were decided on: 38 from policyengine-us 1.755.4, 18 from 2.15.17, 2 from 2.38.6, and the 17 that move on 2.38.6 were re-reviewed and stay excluded); same household facts and requested outputs for every model; answer transport and request shape per model as recorded in the serving-configuration table - 47 models",
"id": "1.1",
"label": "1.1",
- "snapshotLabel": "Snapshot 2026-09-30"
+ "snapshotLabel": "Snapshot 2026-10-10"
},
{
"artifact": {
diff --git a/app/src/lib/predictionStatus.ts b/app/src/lib/predictionStatus.ts
index b0b79fbe..21342b1b 100644
--- a/app/src/lib/predictionStatus.ts
+++ b/app/src/lib/predictionStatus.ts
@@ -84,7 +84,7 @@ const EXCLUDED_REFERENCE_BASIS: Record = {
"which assumes one reading of the unlisted input described above",
reference_engine_defect: "which carries the engine defect described above",
reference_law_published_after_freeze:
- "which uses the engine's projection described above",
+ "which uses the engine's own 2026 figure described above",
};
/**
diff --git a/app/src/lib/referenceEngine.ts b/app/src/lib/referenceEngine.ts
index 13692f7b..01ae3ce5 100644
--- a/app/src/lib/referenceEngine.ts
+++ b/app/src/lib/referenceEngine.ts
@@ -15,8 +15,8 @@ import type { ReferenceExclusion } from "../types";
* the frozen sidecar.
*/
export const ENGINE_UPGRADE_RECHECK = {
- engineVersion: "2.15.17",
- rechecked: 19,
+ engineVersion: "2.38.6",
+ rechecked: 17,
};
function compareVersions(a: string, b: string): number {
diff --git a/app/src/lib/servingSensitivity.ts b/app/src/lib/servingSensitivity.ts
index 72c09299..84fbf342 100644
--- a/app/src/lib/servingSensitivity.ts
+++ b/app/src/lib/servingSensitivity.ts
@@ -6,7 +6,8 @@
* regardless); Claude Fable 5.1 rejects forced tool calls and answers as JSON,
* reasoning in both runs, so its comparison is one of transport. Claude Opus
* 5.5 and Claude Sonnet 5.5 reject forced calls the same way and have no auto
- * re-run. Scores are
+ * re-run. Claude Haiku 5.5 accepts forced calls, so its board row is a
+ * forced-tool row like Claude Opus 5's. Scores are
* the pinned three-decimal measurements from sensitivity/data/*.json; round
* only for display. Each entry's "would rank" is derived from the live board
* rows at render time, never typed by hand.
@@ -35,7 +36,7 @@ const FORCED_TOOL =
export const SERVING_SENSITIVITY: Record = {
"claude-fable-5": {
- autoExact: 92.817,
+ autoExact: 91.746,
boardTreatment: `${FORCED_TOOL}, one output per request`,
autoTreatment:
"re-run with tool_choice: auto and the whole household in one request",
@@ -43,14 +44,14 @@ export const SERVING_SENSITIVITY: Record = {
noteHref: SENSITIVITY_DOC_HREF,
},
"claude-opus-5": {
- autoExact: 91.252,
+ autoExact: 90.456,
boardTreatment: FORCED_TOOL,
autoTreatment: "re-run with tool_choice: auto",
thinkingSuppressedOnBoard: true,
noteHref: SENSITIVITY_DOC_HREF,
},
"claude-sonnet-5": {
- autoExact: 86.076,
+ autoExact: 84.953,
boardTreatment: `${FORCED_TOOL}, one output per request`,
autoTreatment:
"re-run with tool_choice: auto and the whole household in one request",
@@ -58,7 +59,7 @@ export const SERVING_SENSITIVITY: Record = {
noteHref: SENSITIVITY_DOC_HREF,
},
"claude-fable-5.1": {
- autoExact: 93.037,
+ autoExact: 92.044,
boardTreatment:
"rejects forced tool calls, so its row answers as a JSON object and reasons at the provider default",
autoTreatment:
@@ -66,6 +67,13 @@ export const SERVING_SENSITIVITY: Record = {
thinkingSuppressedOnBoard: false,
noteHref: "/notes/2026-09-01-claude-fable-5-1-added",
},
+ "claude-haiku-5.5": {
+ autoExact: 89.239,
+ boardTreatment: FORCED_TOOL,
+ autoTreatment: "re-run with tool_choice: auto",
+ thinkingSuppressedOnBoard: true,
+ noteHref: SENSITIVITY_DOC_HREF,
+ },
};
export function servingSensitivityFor(
diff --git a/app/src/model-serving-config.json b/app/src/model-serving-config.json
index 99dcbe8c..eade7173 100644
--- a/app/src/model-serving-config.json
+++ b/app/src/model-serving-config.json
@@ -13,7 +13,7 @@
},
"evidence_summary": {
"registry": 29,
- "run_state": 17
+ "run_state": 18
},
"models": {
"claude-fable-5": {
@@ -108,6 +108,63 @@
"shared_completion_budget_tokens": null,
"tool_choice": "forced"
},
+ "claude-haiku-5.5": {
+ "answer_contract": "tool",
+ "evidence": {
+ "compared_columns": [
+ "scenario_id",
+ "variable",
+ "prediction",
+ "explanation",
+ "raw_response",
+ "provider_resolved_model",
+ "prompt_tokens",
+ "completion_tokens",
+ "error"
+ ],
+ "fields": [
+ "answer_contract",
+ "chunk_size",
+ "completion_budget_ceiling",
+ "fingerprint_version",
+ "initial_completion_budget_tokens",
+ "max_repair_rounds",
+ "model_id",
+ "prompt_contract_version",
+ "request_timeout_seconds",
+ "thinking",
+ "tool_choice_mode"
+ ],
+ "frozen_rows_sha256": "4a7373791a5f9df6e4c775b8aade37ecd58df87bd8b99f238889f1afe744621c",
+ "kind": "run_state",
+ "rows_match": true,
+ "run": "haiku55-runs/haiku55",
+ "source_predictions_sha256": "27a5e48f44ad875f6d0901571b91c06c67045a7a6841f95a5ab56ecf0cfc8344",
+ "source_rows_sha256": "4a7373791a5f9df6e4c775b8aade37ecd58df87bd8b99f238889f1afe744621c",
+ "treatment_fingerprint": {
+ "answer_contract": "tool",
+ "chunk_size": null,
+ "completion_budget_ceiling": 128000,
+ "fingerprint_version": 3,
+ "initial_completion_budget_tokens": 16384,
+ "max_repair_rounds": 2,
+ "model_id": "claude-haiku-5-5",
+ "prompt_contract_version": "2026-08-09-v2-scoring-contract",
+ "request_timeout_seconds": 300,
+ "thinking": {
+ "mode": "provider_default"
+ },
+ "tool_choice_mode": "forced"
+ }
+ },
+ "provider_id": "claude-haiku-5-5",
+ "reasoning_setup": "provider default; 16,384-token shared budget",
+ "registry_derived": [],
+ "request_shape": "whole scenario",
+ "request_timeout_seconds": 300,
+ "shared_completion_budget_tokens": 16384,
+ "tool_choice": "forced"
+ },
"claude-opus-4.7": {
"answer_contract": "tool",
"evidence": {
@@ -1528,7 +1585,7 @@
"tool_choice": null
}
},
- "registry_commit": "cddd01812ad6d6311502ef997f93634f70c28fda",
+ "registry_commit": "0562d77a9d263c14bb9c9fa93ba3c765fcff95a9",
"sources": [
"policybench.config.MODELS",
"policybench.model_cards",
diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts
index e41407c4..94d27faf 100644
--- a/app/src/modelMeta.ts
+++ b/app/src/modelMeta.ts
@@ -13,6 +13,7 @@ export const MODEL_ORDER = [
"claude-sonnet-5.5",
"claude-sonnet-5",
"claude-sonnet-4.6",
+ "claude-haiku-5.5",
"claude-haiku-4.5",
"grok-4.3",
"grok-4.5",
@@ -60,6 +61,7 @@ export const MODEL_LABELS: Record = {
"claude-opus-5": "Claude Opus 5",
"claude-opus-4.8": "Claude Opus 4.8",
"claude-opus-4.7": "Claude Opus 4.7",
+ "claude-haiku-5.5": "Claude Haiku 5.5",
"claude-haiku-4.5": "Claude Haiku 4.5",
"claude-sonnet-5.5": "Claude Sonnet 5.5",
"claude-sonnet-5": "Claude Sonnet 5",
diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json
new file mode 100644
index 00000000..d8870c82
--- /dev/null
+++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json
@@ -0,0 +1,90 @@
+{
+ "slug": "2026-10-10-claude-haiku-5-5-joins-the-board",
+ "date": "2026-10-10",
+ "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6",
+ "paragraphs": [
+ "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers within $1, weighted by household impact, #{haikuRank} of {nModels} and {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).",
+ "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.",
+ "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.",
+ "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.",
+ "The new version counts Indiana county income tax in local income tax, and the prompt names no county, so PolicyBench stops scoring the local income tax of {indianaOutputs:words} Indiana households. It also counts Idaho's $10 permanent building fund tax in state income tax, which moves one household's reference from ${idahoBefore} to ${idahoAfter}. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.",
+ "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's rate would have risen, by {riseWithoutMin} to {riseWithoutMax} points."
+ ],
+ "facts": {
+ "releaseDate": "2026-10-10",
+ "nModels": 47,
+ "haikuExact": 79.6,
+ "haikuRank": 37,
+ "haiku45Exact": 74.8,
+ "haiku45Rank": 43,
+ "haikuGain": 4.8,
+ "solExact": 95.6,
+ "opusExact": 94.4,
+ "sol56Exact": 93.5,
+ "haikuCost": 0.0013,
+ "haiku45Cost": 0.0554,
+ "haikuAutoExact": 89.2,
+ "haikuAutoRank": 10,
+ "previousEngine": "2.15.17",
+ "engineVersion": "2.38.6",
+ "restored": 18,
+ "restoredRuled": 4,
+ "restoredFixes": "the IRA deduction's compensation limit and a dependent's contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction's active-participant phase-out and California's itemized deduction conformity (policyengine-us #10032, #10034), New Jersey's worker unemployment and workforce contributions (policyengine-us #10031), Arizona's standard deduction indexing (policyengine-us #9928), California's itemized deduction conformity (policyengine-us #10034), Ohio's medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado's 2026 sales tax refund (policyengine-us #9946), the IRA deduction's active-participant phase-out (policyengine-us #10032; four outputs), New York's 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction's active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs)",
+ "adversaryOutputs": 8,
+ "louisianaOutputs": 2,
+ "indianaOutputs": 2,
+ "idahoBefore": "6,818.34",
+ "idahoAfter": "6,828.34",
+ "scoredOutputs": 1926,
+ "totalOutputs": 1984,
+ "excluded": 58,
+ "incumbents": 46,
+ "fallMin": 0.24,
+ "fallMax": 1.37,
+ "newlyScored": 14,
+ "newlyScoredHalfAtMost": 1,
+ "newlyScoredUnmatched": 4,
+ "riseWithoutMin": 0.39,
+ "riseWithoutMax": 0.71
+ },
+ "data": [
+ {
+ "label": "Dashboard data release",
+ "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20261010"
+ },
+ {
+ "label": "Engine upgrade record",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/reference_audit/2026-10-09-engine-upgrade"
+ },
+ {
+ "label": "Reference sidecar with every change",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json"
+ },
+ {
+ "label": "Exclusion record",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json"
+ },
+ {
+ "label": "Reference adversary's record",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/reference_audit/2026-10-05-reference-adversary/README.md"
+ },
+ {
+ "label": "Louisiana standard deduction audit",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/reference_audit/2026-10-05-louisiana/README.md"
+ },
+ {
+ "label": "Serving sensitivity note",
+ "href": "https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md"
+ },
+ {
+ "label": "Claude Haiku 5.5 model page",
+ "href": "/model/claude-haiku-5.5"
+ },
+ {
+ "label": "Previous release note (October 6)",
+ "href": "/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs"
+ }
+ ],
+ "boardSnapshot": "2026-10-10",
+ "release": "dashboard-data-20261010"
+}
diff --git a/app/src/notes/index.ts b/app/src/notes/index.ts
index ae6805ba..58d30bfa 100644
--- a/app/src/notes/index.ts
+++ b/app/src/notes/index.ts
@@ -1,3 +1,4 @@
+import claudeHaiku55JoinsTheBoard from "./2026-10-10-claude-haiku-5-5-joins-the-board.json";
import claudeFable51Added from "./2026-09-01-claude-fable-5-1-added.json";
import claudeSonnet55DebutsFifth from "./2026-09-29-claude-sonnet-5-5-debuts-fifth.json";
import fiveSnapHouseholdsBbce from "./2026-10-05-five-snap-households-bbce.json";
@@ -29,6 +30,7 @@ export type PolicyBenchNote = {
};
export const notes = [
+ claudeHaiku55JoinsTheBoard,
stopsScoringEightTaxOutputs,
fiveSnapHouseholdsBbce,
claudeSonnet55DebutsFifth,
diff --git a/app/src/paperSnapshot.json b/app/src/paperSnapshot.json
index 3f4ba9c1..f8af53e9 100644
--- a/app/src/paperSnapshot.json
+++ b/app/src/paperSnapshot.json
@@ -1,6 +1,6 @@
{
- "pdfVersion": "2cffc69cffde",
- "responseWindow": "June 12 and September 29, 2026",
- "snapshotDate": "2026-09-30",
- "webVersion": "c6dfba0c5718"
+ "pdfVersion": "1607c71abaf3",
+ "responseWindow": "June 12 and October 8, 2026",
+ "snapshotDate": "2026-10-10",
+ "webVersion": "31cd4141373a"
}
diff --git a/app/tests/auditUniverse.test.ts b/app/tests/auditUniverse.test.ts
index 210a1718..23ad1ec9 100644
--- a/app/tests/auditUniverse.test.ts
+++ b/app/tests/auditUniverse.test.ts
@@ -18,16 +18,17 @@ describe("audit universe", () => {
expect(AUDIT_SELECTION_RULE).toBe(
"rows whose legacy threshold score is below 1",
);
- // Release 20261006: the counts tests/test_disclosures.py recomputes
- // from the frozen annotations and payload (release 20260930's less the
- // 368 scored rows on the eight outputs it excludes).
+ // Release 20261010: the counts tests/test_disclosures.py recomputes
+ // from the frozen annotations and payload (47 models; the ten outputs
+ // ruled on 2026-10-06 and the two Indiana county outputs leave scoring,
+ // the outputs the engine upgrade regenerates return to it).
expect(summarizeAuditUniverse(us)).toEqual({
- annotatedRowCount: 7_527,
- legacyThresholdRowCount: 7_527,
- exactMissCount: 7_523,
- annotatedExactMissCount: 7_523,
+ annotatedRowCount: 8_001,
+ legacyThresholdRowCount: 8_001,
+ exactMissCount: 7_997,
+ annotatedExactMissCount: 7_997,
annotatedExactHitCount: 4,
- unannotatedBelowFullBoundedScoreCount: 2_072,
+ unannotatedBelowFullBoundedScoreCount: 2_138,
});
});
});
diff --git a/app/tests/boardScope.test.ts b/app/tests/boardScope.test.ts
index 41e4c668..df8a67a3 100644
--- a/app/tests/boardScope.test.ts
+++ b/app/tests/boardScope.test.ts
@@ -131,10 +131,26 @@ describe("dataset switching", () => {
byEngine.set(version, (byEngine.get(version) ?? 0) + 1);
}
const engine = data.policyengineBundles?.us?.model_version as string;
- const [older] = [...byEngine.keys()].filter((version) => version !== engine);
+ // Oldest engine first, as the sentence lists them.
+ const groups = [...byEngine.entries()]
+ .sort(([a], [b]) => {
+ const left = a.split(".").map(Number);
+ const right = b.split(".").map(Number);
+ for (let i = 0; i < Math.max(left.length, right.length); i += 1) {
+ const diff = (left[i] ?? 0) - (right[i] ?? 0);
+ if (diff !== 0) return diff;
+ }
+ return 0;
+ })
+ .map(([version, count], index) =>
+ index === 0
+ ? `${count} computed with policyengine-us ${version}`
+ : `${count} with ${version}`,
+ );
+ expect(groups.length).toBeGreaterThanOrEqual(2);
const html = render("1.1").replaceAll("'", "'");
expect(html).toContain(
- `The ${exclusions.length} excluded outputs keep the values they were decided on (${byEngine.get(older)} computed with policyengine-us ${older}, ${byEngine.get(engine)} with ${engine})`,
+ `The ${exclusions.length} excluded outputs keep the values they were decided on (${groups.join(", ")})`,
);
expect(html).toContain(`that move on ${engine}; all`);
expect(render("1.0")).not.toContain("excluded outputs keep the values");
diff --git a/app/tests/dataVersions.test.ts b/app/tests/dataVersions.test.ts
index 6a5cb324..5fbb6e04 100644
--- a/app/tests/dataVersions.test.ts
+++ b/app/tests/dataVersions.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import registryJson from "../src/data.versions.json";
+import paperSnapshot from "../src/paperSnapshot.json";
import {
isLivePointerRef,
parseDataVersionRegistry,
@@ -93,7 +94,7 @@ describe("parseDataVersionRegistry", () => {
expect(
registry.versions.find((version) => version.id === registry.default)
?.snapshotLabel,
- ).toBe("Snapshot 2026-09-30");
+ ).toBe(`Snapshot ${paperSnapshot.snapshotDate}`);
});
});
diff --git a/app/tests/expandPage.test.ts b/app/tests/expandPage.test.ts
index c34e98d8..07c8eb07 100644
--- a/app/tests/expandPage.test.ts
+++ b/app/tests/expandPage.test.ts
@@ -71,17 +71,18 @@ test("expand page derives its exact-score headline from the live summary", async
).text();
expect(leader.model).toBe("gpt-6-sol");
- expect(leader.exact).toBeCloseTo(95.828, 3);
+ expect(leader.exact).toBeCloseTo(95.583, 3);
expect(pageSource).toContain("const leader = headlineExactLeader(dashboard);");
expect(pageSource).toContain(
"const leaderLabel = MODEL_LABELS[leader.model] ?? leader.model;",
);
expect(pageSource).toContain("The best model, {leaderLabel}, computes");
expect(pageSource).toContain("{leader.exact.toFixed(1)}% of");
- expect(pageSource).not.toContain("95.8%");
+ expect(pageSource).not.toContain("95.6%");
+ expect(pageSource).not.toContain("96.5%");
expect(pageSource).not.toContain("95.0%");
expect(expectedCopy).toBe(
- "GPT-6 Sol computes 95.8% of requested outputs exactly",
+ "GPT-6 Sol computes 95.6% of requested outputs exactly",
);
});
diff --git a/app/tests/heatmapMetric.test.ts b/app/tests/heatmapMetric.test.ts
index 8c94d2bf..52efa869 100644
--- a/app/tests/heatmapMetric.test.ts
+++ b/app/tests/heatmapMetric.test.ts
@@ -36,8 +36,8 @@ describe("program heatmap metric", () => {
test("exact and bounded rates differ where partial credit is large", () => {
const federal = fable51("federal_income_tax_before_refundable_credits");
- expect(Math.round(heatmapValue(federal, "exact"))).toBe(82);
- expect(Math.round(heatmapValue(federal, "score"))).toBe(94);
+ expect(Math.round(heatmapValue(federal, "exact"))).toBe(80);
+ expect(Math.round(heatmapValue(federal, "score"))).toBe(96);
const stateTax = fable51("state_income_tax_before_refundable_credits");
expect(Math.round(heatmapValue(stateTax, "exact"))).toBe(74);
expect(Math.round(heatmapValue(stateTax, "score"))).toBe(94);
@@ -78,7 +78,7 @@ describe("program heatmap metric", () => {
const boundedRates = entries.map((e) => Math.round(e.score)).sort((a, b) => a - b);
expect(cells).toEqual(exactRates);
expect(cells).not.toEqual(boundedRates);
- expect(cells).toContain(82);
+ expect(cells).toContain(80);
expect(html).toContain('aria-pressed="true"');
});
diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts
index aa789bf3..a10f28c8 100644
--- a/app/tests/notes.test.ts
+++ b/app/tests/notes.test.ts
@@ -9,6 +9,8 @@ import {
interpolateNoteText,
} from "../src/components/NotesContent";
import { notes } from "../src/notes";
+import artifact from "../src/data.artifact.json";
+import paperSnapshot from "../src/paperSnapshot.json";
describe("notes", () => {
test("the index is newest first", () => {
@@ -51,8 +53,8 @@ describe("notes", () => {
"Most models answer $0 for households that qualify for SNAP under their states' higher income limits",
);
expect(markup).toContain("release dashboard-data-20260922");
- // The BBCE note keeps release dashboard-data-20260930; the October 6
- // release note is on the frozen release.
+ // The BBCE note keeps release dashboard-data-20260930, and the October 6
+ // release note its own release.
expect(markup).toContain("release dashboard-data-20260930");
expect(markup).toContain(
"PolicyBench stops scoring eight tax outputs whose references turn on facts the prompts never state",
@@ -77,12 +79,17 @@ describe("notes", () => {
expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*(?::words)?\}/);
});
- test("the newest note is the release note for dashboard-data-20261006", () => {
- expect(notes[0].slug).toBe(
+ test("the newest note is the frozen release's, ahead of the October 6 one", () => {
+ // The live pointer names the frozen release (src/data.artifact.json).
+ expect(notes[0].slug).toContain("claude-haiku-5-5-joins-the-board");
+ expect(notes[0].release).toBe(artifact.tag);
+ expect(notes[0].boardSnapshot).toBe(paperSnapshot.snapshotDate);
+ expect(notes[0].slug.startsWith(notes[0].date)).toBe(true);
+ expect(notes[1].slug).toBe(
"2026-10-06-policybench-stops-scoring-eight-tax-outputs",
);
- expect(notes[0].release).toBe("dashboard-data-20261006");
- expect(notes[0].boardSnapshot).toBe("2026-09-30");
+ expect(notes[1].release).toBe("dashboard-data-20261006");
+ expect(notes[1].boardSnapshot).toBe("2026-09-30");
});
test("a words placeholder spells out whole numbers under ten", () => {
diff --git a/app/tests/programChart.test.ts b/app/tests/programChart.test.ts
index 6c3d437e..223a5870 100644
--- a/app/tests/programChart.test.ts
+++ b/app/tests/programChart.test.ts
@@ -21,7 +21,7 @@ describe("program bars", () => {
const federal = bars.find(
(b) => b.variable === "federal_income_tax_before_refundable_credits",
)!;
- expect(federal.exact).toBeCloseTo(82.3, 1);
+ expect(federal.exact).toBeCloseTo(79.76, 1);
expect(federal.weightShare).toBeCloseTo(0.191, 2);
const total = bars.reduce((sum, b) => sum + (b.weightShare ?? 0), 0);
expect(total).toBeCloseTo(1, 3);
@@ -37,10 +37,10 @@ describe("program bars", () => {
);
expect(html.match(/
82.3%<");
+ expect(html).toContain(">79.8%<");
expect(html).toContain(">19%<");
- expect(html).toContain("82.3% exact");
- expect(html).toContain("width:82.27848101265823%");
+ expect(html).toContain("79.8% exact");
+ expect(html).toContain("width:79.76190476190477%");
});
test("narrow rows stack the bar under the label; wide rows reserve a bar track", () => {
diff --git a/app/tests/referenceEngine.test.ts b/app/tests/referenceEngine.test.ts
index 703f2cd5..25d331ce 100644
--- a/app/tests/referenceEngine.test.ts
+++ b/app/tests/referenceEngine.test.ts
@@ -101,14 +101,17 @@ describe("excluded-output engines against the Python record", () => {
"utf8",
).replace(/<[^>]+>/g, "");
const match = html.match(
- /keep the values they were decided on \((\d+) computed with policyengine-us ([\d.]+), (\d+) with ([\d.]+)\)/,
+ /keep the values they were decided on \((\d+ computed with policyengine-us [\d.]+(?:, \d+ with [\d.]+)+)\)/,
);
expect(match).not.toBeNull();
- const [, olderCount, older, newerCount, newer] = match!;
- expect(excludedOutputsByEngine(exclusions)).toEqual([
- [older, Number(olderCount)],
- [newer, Number(newerCount)],
- ]);
+ const groups = match![1]
+ .replace("computed with policyengine-us ", "with ")
+ .split(", ")
+ .map((group) => {
+ const [count, , version] = group.split(" ");
+ return [version, Number(count)];
+ });
+ expect(excludedOutputsByEngine(exclusions)).toEqual(groups);
});
test("the live sentence names the reference engine's recheck", () => {
diff --git a/app/tests/servingConfig.test.ts b/app/tests/servingConfig.test.ts
index 1ba7a392..fc590d16 100644
--- a/app/tests/servingConfig.test.ts
+++ b/app/tests/servingConfig.test.ts
@@ -8,6 +8,7 @@ import ModelLeaderboard from "../src/components/ModelLeaderboard";
import rawData from "../src/data-summary.json";
import servingConfig from "../src/model-serving-config.json";
import augustSummary from "../../sensitivity/data/claude-thinking-2026-08.json";
+import haiku55Summary from "../../sensitivity/data/claude-haiku-5-5-thinking.json";
import fable51Summary from "../../sensitivity/data/claude-fable-5-1-thinking.json";
import { MODEL_LABELS } from "../src/modelMeta";
import { buildProgramOptions } from "../src/lib/programFilters";
@@ -194,22 +195,25 @@ describe("leaderboard serving copy", () => {
});
test("states the re-run counts the sensitivity data records", () => {
- // The August runs are the forced-tool rows, whose thinking was off; the
- // Fable 5.1 run compares transports on a row that reasons.
+ // The August runs and Haiku 5.5's are forced-tool rows, whose thinking
+ // was off; the Fable 5.1 run compares transports on a row that reasons.
const counts = servingSensitivityCounts();
- const august = Object.values(augustSummary.runs).map((run) => run.model);
+ const forced = [
+ ...Object.values(augustSummary.runs).map((run) => run.model),
+ haiku55Summary.model,
+ ];
expect(Object.keys(SERVING_SENSITIVITY).sort()).toEqual(
- [...august, fable51Summary.model].sort(),
+ [...forced, fable51Summary.model].sort(),
);
expect(
Object.entries(SERVING_SENSITIVITY)
.filter(([, entry]) => entry.thinkingSuppressedOnBoard)
.map(([model]) => model)
.sort(),
- ).toEqual([...august].sort());
+ ).toEqual([...forced].sort());
expect(counts).toEqual({
- reruns: august.length + 1,
- thinkingSuppressed: august.length,
+ reruns: forced.length + 1,
+ thinkingSuppressed: forced.length,
});
const html = render("1.1");
expect(html).toContain(
diff --git a/app/tests/servingSensitivity.test.ts b/app/tests/servingSensitivity.test.ts
index 780d688d..17c21cac 100644
--- a/app/tests/servingSensitivity.test.ts
+++ b/app/tests/servingSensitivity.test.ts
@@ -13,6 +13,7 @@ import {
servingSensitivityFor,
} from "../src/lib/servingSensitivity";
import fable51Summary from "../../sensitivity/data/claude-fable-5-1-thinking.json";
+import haiku55Summary from "../../sensitivity/data/claude-haiku-5-5-thinking.json";
import augustSummary from "../../sensitivity/data/claude-thinking-2026-08.json";
import { wouldRank } from "../src/lib/wouldRank";
import rawData from "../src/data-summary.json";
@@ -37,10 +38,11 @@ function renderFable5Chip(): string {
}
describe("serving sensitivity", () => {
- test("covers exactly the four Claude rows whose request shape changed", () => {
+ test("covers exactly the five Claude rows with an auto re-run", () => {
expect(Object.keys(SERVING_SENSITIVITY).sort()).toEqual([
"claude-fable-5",
"claude-fable-5.1",
+ "claude-haiku-5.5",
"claude-opus-5",
"claude-sonnet-5",
]);
@@ -59,11 +61,11 @@ describe("serving sensitivity", () => {
wouldRank: rank,
}),
);
- expect(rank).toBe(6);
- expect(html).toContain("auto 92.8 · #6");
+ expect(rank).toBe(7);
+ expect(html).toContain("auto 91.7 · #7");
expect(html).toContain("switches Claude's extended thinking off");
- expect(html).toContain("would rank #6");
- expect(html).toContain("(+7.9 against its 84.9% on the unfiltered board)");
+ expect(html).toContain("would rank #7");
+ expect(html).toContain("(+8.1 against its 83.7% on the unfiltered board)");
expect(html).toContain("sensitivity/claude-thinking-2026-08.md");
expect(html).toContain("issues/139");
});
@@ -80,16 +82,36 @@ describe("serving sensitivity", () => {
wouldRank: rank,
}),
);
- // 93.037 ranks above GPT-6 Luna's 93.021 though both display as 93.0:
- // the would-rank comes from the unrounded score.
+ // The would-rank comes from the unrounded score: 93.545 sits between
+ // Claude Sonnet 5.5's 93.575 and GPT-6 Luna's 93.141.
expect(rank).toBe(5);
- expect(html).toContain("auto 93.0 · #5");
- expect(html).toContain("(+1.0 against its 92.1% on the unfiltered board)");
+ expect(html).toContain("auto 92.0 · #5");
+ expect(html).toContain("(+0.8 against its 91.3% on the unfiltered board)");
expect(html).toContain("rejects forced tool calls");
expect(html).toContain("compares transports");
expect(html).toContain('href="/notes/2026-09-01-claude-fable-5-1-added"');
});
+ test("Haiku 5.5 is a forced-tool row whose thinking was off on the board", () => {
+ const sensitivity = servingSensitivityFor("claude-haiku-5.5")!;
+ const board = rows.find((row) => row.model === "claude-haiku-5.5")!;
+ const rank = wouldRank(sensitivity.autoExact, rows);
+ const html = renderToStaticMarkup(
+ createElement(ServingSensitivityChip, {
+ modelLabel: "Claude Haiku 5.5",
+ boardExact: board.exact ?? board.score,
+ sensitivity,
+ wouldRank: rank,
+ }),
+ );
+ expect(rank).toBe(haiku55Summary.sensitivity.would_rank);
+ expect(rank).toBe(10);
+ expect(html).toContain("auto 89.2 · #10");
+ expect(html).toContain("switches Claude's extended thinking off");
+ expect(html).toContain("(+9.7 against its 79.6% on the unfiltered board)");
+ expect(html).toContain("sensitivity/claude-thinking-2026-08.md");
+ });
+
test("the leaderboard keeps the scores in one place and marks the rows", () => {
const source = readFileSync(
new URL("../src/components/ModelLeaderboard.tsx", import.meta.url),
@@ -112,21 +134,25 @@ describe("serving sensitivity", () => {
expect(SERVING_SENSITIVITY["claude-fable-5.1"].autoExact).toBe(
fable51Summary.sensitivity.exact,
);
+ expect(SERVING_SENSITIVITY["claude-haiku-5.5"].autoExact).toBe(
+ haiku55Summary.sensitivity.exact,
+ );
for (const run of Object.values(augustSummary.runs)) {
expect(SERVING_SENSITIVITY[run.model].autoExact).toBe(run.sensitivity.exact);
}
// 88.183 − 86.945 = 1.238 → +1.2; rounding first would have said +1.3.
expect(formatDelta(88.183, 86.945)).toBe("+1.2");
expect(formatDelta(88.2, 86.945)).toBe("+1.3");
- // The pinned Fable 5.1 pair on the frozen board: 93.037 − 92.070 = 0.967.
+ // The pinned Fable 5.1 pair on the frozen board: 92.044 − 91.268 = 0.776.
expect(
formatDelta(fable51Summary.sensitivity.exact, fable51Summary.board.exact),
- ).toBe("+1.0");
+ ).toBe("+0.8");
expect(formatDelta(80.775, 80.8)).toBe("−0.0");
const fable5 = Object.values(augustSummary.runs).find(
(run) => run.model === "claude-fable-5",
)!;
- expect(formatDelta(fable5.sensitivity.exact, fable5.board.exact)).toBe("+7.9");
+ // 91.746 − 83.691 = 8.055 → +8.1.
+ expect(formatDelta(fable5.sensitivity.exact, fable5.board.exact)).toBe("+8.1");
});
test("the open panel stays inside the viewport, horizontally and vertically", () => {
diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md
index b682ce8a..145ee271 100644
--- a/docs/benchmark_card.md
+++ b/docs/benchmark_card.md
@@ -68,13 +68,15 @@ be described as faithful reasoning traces.
## Reference outputs
PolicyBench computes each scored US reference by running
-`policyengine_us.Simulation` from policyengine-us 2.15.17, the newest release
-when PolicyBench began sweeping the references on 2026-09-29 (uploaded 00:23
-UTC). The 64 excluded outputs keep the values they were decided on (52 computed
-with policyengine-us 1.755.4, 12 with 2.15.17), and PolicyBench re-reviewed the
-19 of them that move on 2.15.17; all 19 stay excluded. The manifest also
-records policyengine.py 6.1.2 for provenance. Its certified US bundle carries
-policyengine-us 2.2.1, and policyengine.py does not load beside 2.15.17.
+`policyengine_us.Simulation` from policyengine-us 2.38.6, the newest release
+when PolicyBench began sweeping the references on 2026-10-10 (uploaded 05:39
+UTC). policyengine-us 2.38.6 was still the newest release when PolicyBench
+checked PyPI on 2026-10-10 at 06:53 UTC. The 58 excluded outputs keep the
+values they were decided on (38 computed with policyengine-us 1.755.4, 18 with
+2.15.17, 2 with 2.38.6), and PolicyBench re-reviewed the 17 of them that move on
+2.38.6; all 17 stay excluded. The manifest also records policyengine.py 6.1.2
+for provenance. Its certified US bundle carries policyengine-us 2.2.1, and
+policyengine.py does not load beside 2.38.6.
A scored reference follows from the stated facts and from law published before
PolicyBench froze the references on 2026-07-03. Where policyengine-us projects
@@ -83,11 +85,15 @@ the reference takes the amount published before the freeze or, where none was,
the last one published. Nine publication conventions set those amounts, and each
changes parameter values only: SNAP's October to December months, for example,
hold the FY2026 figures rather than USDA's FY2027 figures of 2026-08-21.
-policyengine-us 2.15.17 counts Maryland county income tax in its state income
-tax; an adapter takes it out again, because PolicyBench's state income tax
-output leaves local tax out. policyengine-us 2.17.0, the newest release when
-PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as
-2.15.17 for all 1,984 outputs under the same conventions and adapter.
+
+PolicyBench first moved the references off the engine version that froze them
+on 2026-09-29, to policyengine-us 2.15.17, the newest release when it began
+sweeping the references that day (uploaded 00:23 UTC). policyengine-us 2.15.17
+counts Maryland county income tax in its state income tax; an adapter takes it
+out again, because PolicyBench's state income tax output leaves local tax out,
+and every later build keeps the adapter. policyengine-us 2.17.0, the newest
+release when PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gave the same
+value as 2.15.17 for all 1,984 outputs under the same conventions and adapter.
Four changes between policyengine-us 1.755.4 and 2.15.17 each moved one scored
reference: New Jersey's child tax credit schedule for 2026 to 2028 (P.L.2026,
@@ -97,29 +103,59 @@ child support received counting as income for school meals (7 CFR
245.6(a)(5)(ii)), and the rounding of New York's Empire State child credit
phase-out. Three federal income tax outputs left scoring (below), and two state
income tax references moved by less than $1. The reference sidecar's
-`engine_upgrade` revision lists every change, and `reference_audit/2026-09-28/`
-records the investigation and the independent review behind each.
+2026-09-29 `engine_upgrade` revision lists every change, and
+`reference_audit/2026-09-28/` records the investigation and the independent
+review behind each.
+
+On 2026-10-10 PolicyBench moved the references again, from 2.15.17 to 2.38.6, a
+release that fixes engine defects behind excluded outputs, and recomputed every
+output under the same conventions and adapter. The new version fixes the
+defects behind 18 outputs. Release 20261006 excluded 14 of them, which return to
+scoring; the other four are the 2026-10-06 defects below, which stay scored.
+Each lands within $1 of its audited corrected value, and the sidecar's
+2026-10-10 revision names the upstream fix behind it: the IRA deduction's
+compensation limit and a dependent's contributions (#10032, #9801; three
+outputs), the IRA deduction's active-participant phase-out (#10032; four
+outputs), the IRA deduction's active-participant phase-out and California's
+itemized deduction conformity (#10032, #10034), the IRA deduction's
+active-participant phase-out and estate income in gross income (#10032,
+#10027; two outputs), California's itemized deduction conformity (#10034),
+estate income in gross income (#10027, #9633; two outputs), New Jersey's worker
+unemployment and workforce contributions (#10031), Arizona's standard deduction
+indexing (#9928), Ohio's medical deduction for health insurance premiums
+(#10020), Colorado's 2026 sales tax refund (#9946) and New York's 2026 child and
+dependent care credit (#9948). The move changes one scored reference beyond the
+$1 tolerance: policyengine-us now counts Idaho's $10 permanent building fund
+tax (Idaho Code 63-3082) in state income tax (#9810). It also moves two Indiana
+households' local income tax onto a county the prompt does not state, and
+PolicyBench excluded both outputs. One more output, a California household's
+state income tax, lands on its recorded corrected value but stays excluded: the
+release's judge found that its reference rests on a further defect, California's
+non-conformity to the federal educator expense deduction (FTB Instructions for
+Schedule CA (540), Section C, line 11), which policyengine-us 2.38.6 still
+allows. `reference_audit/2026-10-09-engine-upgrade/` holds the builder, its
+evidence and the reviewed actions.
## Audit scope
-The frozen US annotations cover 7,527 scored rows selected because their
-legacy threshold score is below 1 (2,444 further annotated rows sit on the 64
+The frozen US annotations cover 8,001 scored rows selected because their
+legacy threshold score is below 1 (2,212 further annotated rows sit on the 58
excluded outputs and are description, not audit). This audit universe contains
-7,523 of the snapshot's 7,523 exact-match misses and four exact hits. Another
-2,072 scored rows have a bounded score below 100 but fall outside the
+7,997 of the snapshot's 7,997 exact-match misses and four exact hits. Another
+2,138 scored rows have a bounded score below 100 but fall outside the
legacy-threshold selection and have no audit annotation. Three judge models
produced the verdicts, all of them board rows: GPT-5.6 Sol through the Codex
-CLI for 298 cases, Claude Opus 5 through the Claude Code CLI for the 116 cases
+CLI for 269 cases, Claude Opus 5 through the Claude Code CLI for the 94 cases
the September 5 additions joined that no later judge re-judged, and Claude Opus
-5.5 for 260 cases the September 22, September 29 and September 30 additions joined or a
-reference revision changed; the
+5.5 for 314 cases the September 22, September 29, September 30 and October 9
+additions joined or a reference revision changed; the
manifest's audit_annotation_artifacts.judge_provenance block carries the tally.
Verdicts change no score. A judge verdict outside the final classes, and every
reference-suspect flag, is resolved by a recorded developer adjudication
(annotations/.../us_adjudications.json), which keeps the judge's
verdict (the case's current verdict.json; a flag an earlier judge run raised is
kept and says so) beside the decision and the reasoning. This snapshot carries
-77: one for each excluded output, one for each flagged reference the
+70: one for each excluded output, one for each flagged reference the
adjudication affirmed or replaced with a regenerated reference, and two for
cases whose misses the judge attributed to the reference applying later law;
the adjudication records them as model errors because that law predates the
@@ -141,19 +177,25 @@ gain fix as first built also applied Wisconsin's capital gain exclusion to the
distributions, which the upstream fix does not; that part is recorded as its
own defect, not fixed upstream.
-Sixty-four outputs in 42 households are excluded from scoring for every model
+Fifty-eight outputs in 42 households are excluded from scoring for every model
(`reference_exclusions.json` beside the frozen references, pinned by the
manifest). The September 22 audit recomputed every reference on
policyengine-us 1.755.4 under each defect's sandbox fix and excluded every
-output a defect not fixed upstream moved by more than a dollar: 28 are recorded
-as engine-defect exclusions across the eleven such root causes (among them the
-IRA deduction's compensation limit and phase-out, estate income, and the
-heat-and-eat SNAP utility allowance that P.L. 119-21 ended for households
-without an elderly or disabled member), and an output such a defect moves that
-was already excluded for an unstated input keeps that record. On
-policyengine-us 2.15.17 each of these outputs keeps the value its defect
-produced, or it moved and a re-review found the defect still present.
-Thirty-six depend on an input the prompt never states, such as whether a
+output a defect not fixed upstream moved by more than a dollar. Of those
+records, 14 remain as engine-defect exclusions. Nine rest on the five root
+causes policyengine-us 2.38.6 does not fix (among them Idaho's subtraction for
+health insurance premiums, elective deferrals in the earned income behind the
+EITC, and the heat-and-eat SNAP utility allowance that P.L. 119-21 ended for
+households without an elderly or disabled member). policyengine-us 2.38.6
+computes four of them at their corrected values; they stay excluded because
+each also moves under an input the prompt does not state. policyengine-us
+2.38.6 computes one more at its recorded corrected value, but its reference
+rests on a further engine defect, found on 2.38.6 and not fixed there, so it
+stays excluded. An output such a defect moves that was
+already excluded for an unstated input keeps that record. On each later
+engine these outputs keep the value they were decided on: the engine gives the
+same value, or it moved and a re-review found the exclusion still holds.
+Forty-two depend on an input the prompt never states, such as whether a
person meets SSI's definition of disability, months of SSDI receipt, weekly
hours worked, the type of survivor benefits, who paid for the coverage behind a
disability benefit, whether an adult tax dependent is the filers' child,
@@ -163,11 +205,15 @@ whether the refunded tax reduced federal tax in the year the household paid it
the federal state and local tax deduction takes as its state income tax part
and policyengine-us estimates from federal adjusted gross income, Medicare
enrollment and a Part B premium, which policyengine-us assumes for every
-Medicare-eligible person and counts as a medical expense, or whether the
+Medicare-eligible person and counts as a medical expense, whether the
employer deducts the employee share of a state paid-leave or disability
-premium that the law lets it deduct but does not require; each was recomputed under the other reading a careful
-reader could take, on the engine version that produced its reference, and it
-moved. One of them, a California household head's Medicaid eligibility, moved
+premium that the law lets it deduct but does not require, whether a household's
+income tax covers only the head and spouse's return or every return its members
+must file, or the county of residence, which decides Indiana's county income
+tax; each was recomputed under the other reading a careful reader could take,
+on the engine version that produced its reference, and it moved. The other two
+are Louisiana households' state income tax, whose reference rests on a standard
+deduction amount Louisiana published after the freeze. One of the forty-two, a California household head's Medicaid eligibility, moved
only once the engine read SSI's definition of disability where the law does.
The head's income is above the limit for the adult expansion group, so only a
disability pathway leads to Medi-Cal, and California's Working Disabled Program
@@ -176,9 +222,9 @@ requires SSI's definition of disability (42 CFR 435.540(a)). policyengine-us
reading; with the program's test reading SSI's definition of disability, the
head qualifies only by meeting it.
Exclusion is symmetric: rows that matched the frozen reference leave the score
-with rows that did not, so every model is scored on 1,920 of its 1,984
+with rows that did not, so every model is scored on 1,926 of its 1,984
requested outputs. The rows on those outputs stay annotated as description:
-each carries its exclusion's class, except the 63 answers that never parsed,
+each carries its exclusion's class, except the 48 answers that never parsed,
which stay parse_contract_failure; no scored row carries a descriptive class.
The prompt states disability as one general fact
(any of the six Current Population Survey disability-difficulty items); SSI,
@@ -222,6 +268,26 @@ four households in Minnesota, Colorado, Massachusetts and New York. They also
move two federal income tax outputs that were already excluded, and no output
that is still scored by any amount. No judge had flagged the eight.
+On 2026-10-06 PolicyBench ruled to exclude ten more outputs. Eight came from
+a reference adversary (`reference_audit/2026-10-05-reference-adversary/`), which
+flags the scored outputs where many models, or several of the strongest, agree
+on an answer the reference does not give. A judge that cannot yet see the
+engine's derivation works each flagged output from primary law, and only then
+reconciles its answer with the derivation. The adversary flagged 61 outputs and
+ran 52, the other nine belonging to other audits, and it held 45 references. An
+independent check of its seven other verdicts confirmed four as engine defects
+(Arizona's 2026 standard deduction left unindexed, Ohio's medical deduction for
+health insurance premiums, Colorado's 2026 sales tax refund, and New York's
+2026 child and dependent care credit), refuted two (the Medicaid eligibility of
+two children in a North Carolina household, whose references stand), and found
+the seventh to be a question of definition: where a dependent must file their
+own return, the output definitions do not say whether the household's income
+tax includes it, which reaches four outputs in a Pennsylvania and a Missouri
+household. The other two are the Louisiana outputs above
+(`reference_audit/2026-10-05-louisiana/`). The 2026-10-10 engine move fixed
+the four defects, so those four outputs stay scored; the other six are
+excluded.
+
Canonical runs require numeric answers and explanations for each requested
output. If future prompt-contract ablations omit explanations, they should be
labeled as ablations and not mixed into leaderboard claims.
@@ -297,7 +363,7 @@ Discipline for private files:
- Run evaluations on the private split by passing the private manifest
explicitly (`--scenario-manifest .../scenarios-private.csv`); the eval and
analyze commands need no other changes.
-- Activation is a snapshot decision: the current 2026-09-30 snapshot scores
+- Activation is a snapshot decision: the current 2026-10-10 snapshot scores
100 public households whose scenario manifest was generated on 2026-06-12
from a 125-household request split with seed 1042. It does not report
protected scores. The first snapshot that reports protected scores should
diff --git a/docs/haiku55/design.md b/docs/haiku55/design.md
new file mode 100644
index 00000000..65cbab9e
--- /dev/null
+++ b/docs/haiku55/design.md
@@ -0,0 +1,509 @@
+# Claude Haiku 5.5, the 2026-10-06 exclusions and the engine upgrade: design
+
+## What the release is
+
+This release does three things on top of release `dashboard-data-20261006`:
+
+1. It adds Claude Haiku 5.5 (display id `claude-haiku-5.5`, provider id `claude-haiku-5-5`, run slug `haiku55`) as the board's 47th model.
+2. It installs ten exclusions under Max's rulings of 2026-10-06: d1022 (eight outputs) and d994 (two Louisiana outputs). Each output leaves scoring for every model.
+3. It moves the references from policyengine-us 2.15.17 to a newer release that fixes engine defects behind exclusions (Max, 2026-10-09: "yes i want to wait for hte fixed engine"). An excluded output whose defect the new engine fixes is regenerated and scored again. "The engine upgrade" describes this step.
+
+The driver is `scripts/finish_haiku55.py`, adapted from `scripts/finish_gpt61sol.py`. The freeze is `scripts/freeze_haiku55.py`, adapted from `scripts/freeze_gpt61sol.py`. The inputs are in `docs/haiku55/`:
+
+- `spec.json`: the rulings, the two pinned proposal files, the ten adjudications, the ruled outputs an engine upgrade regenerates (`regenerated_by_upgrade`) and the decisions on the outputs it newly excludes (`upgrade_adjudications`);
+- `input_pins.json`: the finished run;
+- `seed_digest.csv`: the audit seed.
+
+The engine upgrade's builder, actions, evidence and corrected narratives are in `reference_audit/2026-10-09-engine-upgrade/`.
+
+The stage is built in two parts. The first folds the model in and installs the ten records on release 20261006's references. `--step install-references` then installs a reference build on the new engine, and every later step holds the stage to that build. The table gives the first part's counts, which the driver enforces until a build is installed.
+
+| | Release 20261006 | The stage before a build is installed |
+|---|---|---|
+| Models | 46 | 47 |
+| Requested outputs per model | 1,984 | 1,984 |
+| Exclusions | 64: 36 unlisted inputs, 28 engine defects | 74: 40 unlisted inputs, 32 engine defects, 2 later-published law |
+| Scored outputs per model | 1,920 | 1,910 |
+| Adjudications | 77, of which 64 exclude | 86, of which 74 exclude |
+
+Sources: the base's exclusion and adjudication records at `BASE_COMMIT`, and the release's record as `build_release_exclusions` builds it. The ten new decisions are nine new entries plus scenario_051's existing entry, which is restated. The 86 holds when triage adds no other decision; it may add one only on a re-opened case, and only one that keeps the output scored.
+
+An installed build changes these counts. It removes the exclusion records it regenerates, adds the records it newly excludes, and drops the decisions of the release 20261006 records it regenerates. The release's own counts are in the frozen exclusion record, the adjudication record and the reference sidecar's last `engine_upgrade` revision.
+
+What never changes:
+
+- the 100 households and the 1,984 requested outputs;
+- `scenarios.csv` and its sidecar, which stay on their 20261006 pins at every step;
+- every incumbent's predictions and serving row;
+- the judge template (`policybench/audit.py`) and the grounding.
+
+Without an installed build, these also keep their 20261006 bytes:
+
+- every reference value: `reference_outputs.csv` and its sidecar stay on their 20261006 pins;
+- every reference explanation: the freeze requires the staged `us_case_reference_explanations.csv` to be release 20261006's bytes from git;
+- the 64 earlier exclusion records, in their order;
+- every earlier adjudication's decision, except scenario_051's, which d994 restates.
+
+An installed build changes them only as its revision lists ("The engine upgrade").
+
+For the incumbents, only `modelStats` changes, and only as the exclusion records and the installed build explain. `build_payload` checks this. With release 20261006's exclusion record put back, every incumbent's entry must be release 20261006's, byte for byte (the scope check). With a build installed, the check also puts back release 20261006's references (`export_on_base_references`). `modelStats` holds only scores and usage (`score`, `exact`, `n`, `costUsd` and so on) and no audit text, so re-judged verdicts cannot move it. Fable 5's usage is still carried from the base (`CARRIED_USAGE`), as in the GPT-6.1 Sol release.
+
+The new tag is `RELEASE_TAG` in `scripts/finish_haiku55.py`, and it is provisional. This note does not spell it out, just as `docs/gpt61sol/design.md` does not spell out its own tag. `spec.json`'s `release_tag` must match the tag (`release_spec`). The export receipt binds the tag, so run `--step export` again after any change to it. The snapshot date is the spec's `snapshot_date`.
+
+## The base
+
+The base is pinned in `scripts/finish_haiku55.py`. Each value below is the one git holds at `BASE_COMMIT`.
+
+| Item | Value |
+|---|---|
+| Base tag, payload sha256 | `dashboard-data-20261006`, `aa34e5c9…d462` (127,225,036 bytes). The `data.json.gz` committed at `BASE_COMMIT` rewraps to it, and the pointer at that commit names it. |
+| Models, outputs, exclusions, scored | 46, 1,984, 64 and 1,920 (every `modelStats` row has `n` 1,920) |
+| `reference_outputs.csv` | `e8bbba8f…2466` |
+| `reference_outputs.csv.meta.json` | `816fef53…1a4b` |
+| `reference_exclusions.json` | `92741dfd…3815` |
+| `scenarios.csv` | `71b16212…858a` |
+| `scenarios.csv.meta.json` | `03a66e90…aebb` |
+| `BASE_COMMIT` | `9ce4ade8…`, the squash merge of PR #202 on main (2026-10-09). |
+| `SEED_RELEASE_COMMIT` | `8b4c0ca1…`, the merge of PR #187 (release 20260930). Its annotations rendered the seed's prompts. |
+| Grounding | `b1e4a9bc…b55c`, the main clone's `results/local/unified_audit/grounding.csv` |
+| Seed digest | `c780af63…8430` (`docs/haiku55/seed_digest.csv`, 674 judged cases) |
+
+The seed is release 20261006's audit. That release carried release 20260930's audit over without re-rendering it. Both audits (`release-exclusions-batch/results/local/release-batch/dashboard-data-20261006-r3/audit` and `…/base-20260930/audit`) give the committed digest: 674 judged cases out of 986. `results/local/release-haiku55/seed-20261006` is a read-only copy whose `cases.jsonl` is the 20260930 audit's. The stage's `stage.json` binds that digest.
+
+## The run and onboarding
+
+Sources:
+- `~/reviews/policybench-haiku-5-5/README.md` (the README);
+- the card in `policybench/model_cards.py`;
+- the run directories under `/Users/maxghenis/PolicyEngine/policybench/results/local/`.
+
+- **Card.** PR #201 (head `2f58a357`, merged as `ed4f89a6`) added the card, the config entry, the price and the release date.
+- **Contract.** The Models API lists `claude-haiku-5-5` with `created_at` 2026-10-07 and `max_tokens` 128,000. Unlike Sonnet 5.5 and Opus 5.5, this model accepts forced tool use. The board's rule is a forced answer tool unless the provider rejects it, so the card's `answer_contract` is `tool`.
+- **Thinking.** A request with no thinking parameter returned a thinking block (119 of 123 output tokens). The forced-tool probe returned none. So the row answers without extended thinking, as Claude Opus 5's row does (the card's notes). The run's treatment fingerprint (`run_state.json`) records `answer_contract` `tool`, `tool_choice_mode` `forced` and `thinking.mode` `provider_default`.
+- **Gauntlet.** The three-variable probe parsed 3/3 in 2 s. The full-scenario probe parsed 16/16 in 4 s. Both ran on the tool contract.
+- **Board run.** The command was `policybench run --model claude-haiku-5.5 --max-workers 5 --budget-usd 25`, at `2f58a357`, on the frozen 100-household manifest.
+ - It ran on 2026-10-08 from 21:41 to 21:47 UTC.
+ - It finished 100 of 100 scenarios with 1,984 rows: no missing prediction, no missing explanation, no error row.
+ - It cost $0.13. `run_state.json` records `spent_usd` 0.1274; the rows' `total_cost_usd` sums to $0.1299, with no estimated costs.
+ - The median latency per household is 5.1 s: the median over households of the rows' summed `elapsed_seconds`, 5.11 s in the README.
+ - The retry check (`response_retries/round_1`) found no unit to retry.
+- **Pinned copy.** `haiku55-runs/haiku55/run` holds the run as it finished. `docs/haiku55/input_pins.json` pins its `predictions.csv` (`27a5e48f…8344`) and `run_state.json` (`42f82f50…bd4e`). Both equal `haiku55-stage/run`'s, and `stage.json` and `model-provenance.json` record the same predictions hash.
+- **Sensitivity re-run.** A second run used `POLICYBENCH_TOOL_CHOICE=auto`, following `sensitivity/claude-thinking-2026-08.md` (`launch-auto-run.sh`, budget $10). Its purpose is to let the row carry the same auto chip as Opus 5 (README).
+ - It ran from 21:42 to 21:52 UTC and finished 100 of 100 for $0.27. `haiku55-stage/auto-run/run_state.json` records `spent_usd` 0.2702 and `tool_choice_mode` `auto`.
+ - It is not the board row.
+
+## The judge plan
+
+**What is re-judged.** `prepare` re-renders every case against the 47-model bundle and compares each prompt with the seed's. The stage's `prompt-changes.json` lists:
+- 445 kept cases;
+- 229 changed cases;
+- 2 new cases.
+
+So 231 cases are re-opened and need an Opus 5.5 verdict (`prepare.log`: "445 prompts unchanged, 229 changed and 2 new; 231 cases need Opus 5.5"). The stage's `audit/cases.jsonl` holds 988 cases: these 676, plus 312 parse-failure-only cases that no judge reads.
+
+- All 231 re-opened cases list Claude Haiku 5.5 among their wrong models.
+- The two new cases are outputs that only Claude Haiku 5.5 answers wrong: scenario_012 `child1_head_start_eligible` and scenario_067 `local_income_tax`.
+- All ten ruled outputs are among the 229 changed cases, so each gets a new verdict in this stage for its adjudication to carry.
+- Up to 47 models are wrong in one case, so a verdict must cover up to 47.
+
+An installed build re-opens more cases. `install-references` renders the audit again with the build's references and explanations, sets aside every verdict whose prompt changes, and lists those cases as pending ("The engine upgrade"). These counts are the stage's before that step.
+
+**The nine reworded cases.** A prompt renders the template, the reference value, the reference derivation (the case's reference explanation), the grounding, the question and each wrong model's answer and reasoning (`render_case_prompt`). It renders no exclusion, row annotation, case note or adjudication.
+
+Release 20261006 reworded nine reference explanations after the seed was rendered: eight through its `annotation_rewrites.json` and scenario_031's through PR #197's `reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json`. Its driver renders no prompt, so it carried the audit over unchanged. Those nine prompts therefore differ from the seed's whoever joins them.
+
+`reworded_since_seed()` derives the nine from git by comparing `us_case_reference_explanations.csv` at `SEED_RELEASE_COMMIT` and at `BASE_COMMIT`. It refuses unless they are exactly `REWORDED_SINCE_SEED`:
+
+- scenario_031 `head_medicaid_eligible`;
+- `payroll_tax` for scenario_032, 043, 081 and 082;
+- scenario_077 state income tax;
+- scenario_081 federal income tax;
+- scenario_114 federal and state income tax.
+
+`check_prompt_changes` lets these nine change without the new model and refuses any other incumbent-only change. It also refuses if any of the nine fails to show a changed prompt. On this stage the allowance admits nothing extra. Claude Haiku 5.5 answers all nine wrong, so each is re-judged as a case it joins.
+
+**Isolation.** Each judge runs through `scripts/run_audit_claude.sh`, as the GPT-6.1 Sol stage's isolated judges did:
+- from an empty directory outside any git repository;
+- with no tools and an allowlisted environment;
+- with its transcript kept beside the verdict and checked.
+
+The stage's runner log (`results/local/release-haiku55/judge-batch-1.log`) records `model=claude-opus-5-5 effort=xhigh parallel=2` on Claude Code 2.1.284. It also records a token login (`method` `oauth_token`, no account, no organization).
+
+- The login is the `pb-judge` setup-token login, Max's opt-in of 2026-09-30. The runner is given the declared account `claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30`, and each sidecar records it. This login judged 42 of the GPT-6.1 Sol stage's isolated verdicts (group `isolated: setup-token 2` in `docs/gpt61sol/judge_provenance.json`).
+- That login reached its weekly limit during this release's judging. The verdicts judged after that ran the same way on Subfleet lane claude-18's token, with the declared account `claude:max@axiom.org (subfleet lane claude-18)`. `PROVENANCE_GROUPS` maps each declared account to its group, and a verdict whose sidecar declares any other account has no group and is refused (`judge_provenance_record`).
+- `CLAUDE_CONFIG_DIR` is `~/.policybench-judge/pb-judge-config`. The runner requires that it not be the desktop login's `~/.claude`. The desktop opt-in (`JUDGE_ALLOW_DESKTOP_LOGIN`) stays unset.
+- `AUDIT_PARALLEL` is 2. `AUDIT_EFFORT` is unset, so each judge runs at the runner's default, `xhigh`. The driver's own `--step judge` runs one judge at a time (`AUDIT_PARALLEL=1` in `judge`).
+- The first batch (`launch-judges-batch1.sh`, `AUDIT_ONLY` from `judge-batch-1.txt`) names 221 cases: the 231 re-opened cases minus the ten ruled ones. The ten need their verdicts before `adjudicate-exclusions`.
+
+**Why the judge template does not change.** `policybench/audit.py` is the same blob (`0c947b1d`) at this branch's head, at `BASE_COMMIT`, at `SEED_RELEASE_COMMIT`, at release 20260929's `d616e67c`, on main, and at PR #200's head.
+
+PR #200 removed the template's sentence saying earlier audits' bugs "were fixed before this run" (`bec3b244`). It then restored the sentence in the same PR (`738d6a69`), because removing it changes all 674 seed prompts. This driver carries a verdict over only when its prompt is the seed's byte for byte. Removing the sentence would therefore re-open every judged case or stop the fold. This branch takes only #200's evidence (`4a87c59c`). So the 445 carried verdicts and the 231 new ones all answer one template. The sentence's removal waits for a versioned judge template in a separate PR.
+
+**Provenance record.** Export requires `docs/haiku55/judge_provenance.json` (`verify_judge_provenance`). It holds one entry per re-opened case, with:
+- the staged verdict's and prompt's sha256;
+- its group and its isolation;
+- its sidecar's fields;
+- no e-mail address.
+
+`--step provenance` writes it from the stage's sidecars and checks it (`write_judge_provenance`). Run it after the last judge pass and commit the record before export. Its `counts` give the verdicts in each group.
+
+**Earlier wording.** Triage rebuilds every row annotation and case note from three inputs: the verdicts, the staged adjudications and this stage's `wording-amendments.json`. An amendment may name only a re-opened case (`load_amendments`). Three records hold earlier corrections to that text:
+- the committed `us_wording-amendments.json`: 394 amendments, of which 393 are in re-opened cases. The other one is scenario_067 SSI's reasoning, which the adjudication record keeps as committed;
+- release 20261006's `annotation_rewrites.json`: 306 rewrites, all in re-opened cases (288 row annotations, 10 case notes, 8 explanations);
+- PR #197's `rewrites.json`: 45 rewrites, all for scenario_031 `head_medicaid_eligible`, which is re-opened.
+
+In the re-opened cases, the new verdicts' text replaces the corrected text. Review it against these corrections, and list in `$PB_STAGE/wording-amendments.json` any correction that is still needed. When the stage lists any amendment, the freeze commits the stage's list in place of release 20261006's record (`committed_amendments`).
+
+## The ten exclusions
+
+Sources:
+- `docs/haiku55/spec.json`;
+- `reference_audit/2026-10-05-reference-adversary/proposed_changes.json` (sha256 `3a6e5920…49bf`, as #200 merged it in 4db91b5f);
+- `reference_audit/2026-10-05-louisiana/proposed_exclusions.json` (sha256 `a78120cd…2545`).
+
+Values are rounded to cents; the records hold them in full. "Income tax" means the `…_income_tax_before_refundable_credits` output.
+
+| Output | Ruling | Reason code (root cause) | Frozen | Alternative |
+|---|---|---|---|---|
+| scenario_018 (AZ) state income tax | d1022 | `reference_engine_defect` (`az_standard_deduction_indexing`) | 1,146.05 | 1,137.30 |
+| scenario_025 (OH) state income tax | d1022 | `reference_engine_defect` (`oh_medical_deduction_premiums`) | 1,921.57 | 1,916.60 |
+| scenario_043 (CO) state refundable credits | d1022 | `reference_engine_defect` (`co_sales_tax_refund_surplus`) | 19.00 | 0.00 |
+| scenario_082 (NY) state refundable credits | d1022 | `reference_engine_defect` (`ny_cdcc_606_c2`) | 667.00 | 1,187.61 |
+| scenario_093 (MO) federal income tax | d1022 | `reference_depends_on_unlisted_input` (`household_scope_dependent_returns`) | 8,628.99 | 11,848.99 |
+| scenario_093 (MO) state income tax | d1022 | `reference_depends_on_unlisted_input` (same) | 3,388.25 | 4,528.08 |
+| scenario_123 (PA) federal income tax | d1022 | `reference_depends_on_unlisted_input` (same) | 5,200.24 | 8,420.24 |
+| scenario_123 (PA) state income tax | d1022 | `reference_depends_on_unlisted_input` (same) | 3,070.06 | 4,451.56 |
+| scenario_051 (LA) state income tax | d994 | `reference_law_published_after_freeze` (`la_2026_standard_deduction_unpublished`) | 820.35 | 820.26 |
+| scenario_077 (LA) state income tax | d994 | `reference_law_published_after_freeze` (same) | 305.14 | 305.05 |
+
+Each record is dated 2026-10-06, decided by the developer and computed on policyengine-us 2.15.17. `spec_records` refuses any record that is not.
+
+**The rulings** (`spec.json` `rulings`):
+
+- d1022 excludes the reference adversary's four confirmed engine defects (verdict `CONFIRMED` in the proposal file).
+ - These four are regenerated once a policyengine-us release fixes them. The spec's `regenerated_by_upgrade` names them with their upstream fixes (PolicyEngine/policyengine-us#9928, #10020, #9946 and #9948), and an installed build must regenerate exactly these four of the ten ("The engine upgrade").
+ - d1022 also excludes the four outputs that rest on the dependent's own return, the one root cause the adversary left `AMBIGUOUS`. They stay excluded until d1029 settles the household scope for version 2.
+ - The adversary's scenario_026 finding was refuted, and those references stand.
+- d994 excludes the two Louisiana outputs. It keeps the published-amounts convention for every other reference: Idaho (076), SNAP (008, 038, 109) and Maryland (068) do not change.
+
+**Why the Louisiana pair is later-published law.** These are the first two records with this reason code; release 20261006's 64 records use only the other two. Neither of the other codes fits:
+
+- The prompt states every fact the computation uses, so no input is unlisted.
+- The engine did not misapply the law. R.S. 47:294(B) multiplies 2025's $12,500 by the calendar-2025 CPI-U increase. PolicyEngine/policyengine-us#8411 computed $12,835 on 2026-05-24 from published CPI-U values ($12,500 × 324.054 / 315.605, rounded to the dollar). The records call that "a computation under the statute, not a forecast."
+
+The statute sets no rounding rule and gives the Department no duty to publish the result:
+- Before the 2026-07-03 freeze, Louisiana had published only its withholding and estimated-tax amount, $12,875, and said the amount allowed on 2026 returns would differ.
+- Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28, after the freeze.
+
+So the reference rests on a figure that no Louisiana publication stated before the freeze, and the official figure came out after it. The alternative values are the references under $12,838.
+
+The records' notes add evidence. The sweep reproduces all 1,928 references release 20260930 scored at $12,835 and moves only these two outputs. For each output, none of release 20260930's 46 models answered within $1 of the frozen value or of the value under $12,838. Under $12,500, 2025's amount, the values are $830.40 and $315.19, and 21 and 19 models answered within $1 of them.
+
+**How the classes map.** `EXCLUSION_REASON_CODES` gives each reason code its adjudicated source and reference verdict. The spec gives the subtype, and `exclusion_entry` takes the reference basis from the record.
+
+| Reason code | Adjudicated source | Reference verdict | Subtype | Reference basis |
+|---|---|---|---|---|
+| `reference_engine_defect` | `reference_engine_defect` | `engine_defect` | `taxable_income_or_deductions` (018, 025); `state_local_rule` (043, 082) | the record's `law` |
+| `reference_depends_on_unlisted_input` | `prompt_ambiguity` | `unlisted_input` | `household_unit_or_filing_status` | the record's `unlisted_input` |
+| `reference_law_published_after_freeze` | `reference_later_law` | `later_law` | `taxable_income_or_deductions` | the record's `published` |
+
+**How they are installed.**
+- `build_release_exclusions` takes release 20261006's record from git and adds the ten records, sorted by scenario and variable. They go after the earlier waves' records and before the audit's trailing scenario_023 record.
+- It inserts the spec's `derivation_insert` sentence before the 2026-09-29 marker.
+- Each proposal file must be the bytes the spec pins and must hold exactly the ruled outputs.
+- `install-exclusions` writes the result into the stage's `scoring/` and bundle copies and rebinds it in `stage.json` (`exclusions_installed`). Judge prompts do not render exclusions, so the step can run before or after judging.
+- The spec's `record_edits` then name each engine defect's upstream fix in the record's `upstream` field (PolicyEngine/policyengine-us#9928, #9946 and #9948, merged 2026-10-09; Ohio's #10020, open, with #9925 related), replacing the adversary's "to be filed". The installed record's sha256 is `release_exclusions_sha256()`, which `stage.json`'s `exclusions_installed` must match.
+
+**How they are adjudicated.** `adjudicate-exclusions` runs after the judge and the restate script.
+
+- Nine outputs get a new entry, in release 20261006's field order (`ENTRY_FIELDS`). Each carries its case's bound Opus 5.5 verdict from this stage.
+- scenario_051 already has a decision: its reference was regenerated on 2026-09-22, and it was scored until this release. Its entry is restated in place:
+ - the ruling's decision fields replace the old ones (`llm_error`/`thresholds_rates`/`regenerated` become `reference_later_law`/`taxable_income_or_deductions`/`later_law`);
+ - `excluded_from_scoring` goes in after `adjudicator`;
+ - the reasoning keeps the earlier text ahead of the ruling's.
+- The record's date conventions name the 2026-10-06 wave and the day its decisions were written (`spec.json` `adjudications_written_on`).
+
+Run the step once, after the last judge pass. A second run is refused. A new entry copies its case's verdict as it stands, and triage refuses any entry that no longer matches the bound verdict. After this step the restate script stops on the new entries, which name their current verdicts and carry no `judge_previous`. To re-judge a case later, put the staged `us_adjudications.json` back to release 20261006's, then run the restate script and this step again.
+
+With a build installed, the step decides only the ruled outputs the build keeps excluded. It also decides the outputs the build newly excludes and drops the decisions of the release 20261006 records the build regenerated ("The engine upgrade").
+
+## The engine upgrade
+
+**The ruling.** Release 20261006's references were computed on policyengine-us 2.15.17. Max, 2026-10-09: "yes i want to wait for hte fixed engine" (`spec.json` `rulings`). The release moves the references to a newer policyengine-us release that fixes engine defects behind exclusions. An excluded output whose defect the new engine fixes is regenerated: its record is removed and it is scored again at the engine's value. Every other excluded output keeps the value it was decided on (rule 5).
+
+**Which engine.** References come from the newest policyengine-us release when the reference sweep begins, and at publication PolicyBench checks that the newest release gives the same values (`ENGINE_RULE` in the builder). `reference_audit/2026-10-09-engine-upgrade/scripts/sweep_timing.py` records both:
+
+- `sweep` reads PyPI and records each release's wheel upload time from the reference engine onward, the newest release at the read, and when the sweep wrote its first output. It refuses unless the reference engine was the newest release when the sweep began.
+- `check` reads PyPI again before publication. It takes the reference build and refuses one that is not the release's: its sidecar must name the sweep's engine, pin its CSV and name the committed builder, its reference CSV, sidecar and exclusion record must be the committed snapshot's, and its `computed.csv` must give every scored reference, every value finite. When a newer release is out, `check` runs the check sweep itself, a first pass of this audit's builder in a venv holding that release (the builder refuses unless the venv holds it), then compares every output with the reference engine's, and counts scored and excluded outputs apart. Each input is pinned in the record by sha256. A newer release may move outputs that stay excluded. If it moves a scored output, `check` writes the record and exits non-zero, `engine_upgrade_timing_sentence` in `policybench/paper_results.py` refuses, so the paper does not render, and `tests/test_disclosures.py` fails.
+- Both refuse a sweep whose output does not follow its engine: the wheel's upload, the install and the first output must come in that order. Both also check that the `policyengine_us` the venv imports is the release's wheel: its version, imported from the installed package, not an editable install, every package file at the hash the wheel's RECORD lists (sha256, sha384 or sha512), and no unrecorded importable file (module, sourceless bytecode or native extension) beside them. The check runs in the same process as the builder (`ENGINE_RUNNER`, through `run_on_engine`), launched with `-P` and a fresh, empty bytecode-cache prefix, so no stale bytecode or script-directory package can stand in for the verified source. `sweep_timing.py run` runs the reference build the same way, into a fresh out-dir with a fresh receipt, and refuses unless the builder writes a new `computed.csv`. The check refuses any symlink in the installed package, so installs must be copies (uv's default link modes; not `--link-mode=symlink`). The record keeps the check. A release whose wheels are all yanked is left out, and the record lists any such release newer than the reference engine.
+
+Both write into `reference_audit/2026-10-09-engine-upgrade/verification/`.
+
+**The build.** `reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py` reads release 20261006's references from git at `BASE_COMMIT`, recomputes all 1,984 outputs on the installed engine and writes four files: `reference_outputs.csv`, its sidecar, `reference_exclusions.json` and `reference_traces.json`. The sidecar keeps every earlier revision and gains one `engine_upgrade` revision. The builder takes an actions file in which every output that moves is decided:
+
+| Action | What it decides | What the builder requires |
+|---|---|---|
+| `approved` | A scored output may move. | The engine gives the approved value within half a cent. |
+| `regenerated_exclusions` | An excluded output with reason code `reference_engine_defect` is scored again. | The engine lands within the action's tolerance (at most $1; a flag must be equal) of the audited corrected value, which the action's `target` says how to know. `upstream` names the fix. |
+| `new_exclusions` | A scored output that the new engine moves onto an input the prompt does not state leaves scoring. | A complete exclusion record, frozen at the engine's value, on the build's engine and day. |
+| `excluded_rechecked` | An excluded output moves and stays excluded. | It keeps the value it was decided on. |
+| `kept_exclusions_from_release` | The ruled outputs that stay excluded. | With the regenerated ruled outputs, they are exactly the spec's ten. |
+
+The builder refuses an unexplained move, scored or excluded, and an action that does not describe what the engine does. `reference_audit/2026-10-09-engine-upgrade/actions_from_cells.py` drafts the actions from a first compute pass, with each regeneration's upstream fix taken from `upstreams.json` beside it. The builder refuses a draft, so a reviewer decides every action before the build.
+
+**Regeneration targets.** A regenerated output must land on its audited corrected value. A target says how that value is known:
+
+- `record`: the exclusion record's `alternative_value`. This is right when nothing else in the engine has moved the output since the record was decided.
+- `fix_modules`: the value the record's audited fix modules (`reference_audit/2026-09-22/fixes`, at the bytes committed at `BASE_COMMIT`) give on a recent engine that still has the defect. A committed evidence file, written by the builder's evidence mode on that engine, holds the output there without and with the modules. The modules must move it beyond the tolerance there, which shows the defect was present. On the new engine the builder applies the modules again, and they must move the output by no more than the tolerance, which shows nothing is left to fix.
+
+A fix module can load sibling files, and those run too. `policybench/fix_module_closure.py` finds them from the source: it follows `Path(__file__).with_name("")` and refuses a module that reaches local files any other way. The builder and the driver pin every module in the closure by sha256.
+
+**Narratives.** `reference_audit/2026-10-09-engine-upgrade/scripts/narratives_upgrade.py` rewrites the reference explanation of each output the build changed and keeps every other row's bytes. The writer is the one every reference narrative uses (`policybench.case_reference_explanations`), given the build's engine trace and the change's recorded basis. A narrative must state the new reference value. `reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.json` replaces the writer's text for the outputs it lists, and `hand_corrected_narratives.md` beside it gives the basis of each correction. With `--reuse-from`, an output whose writer inputs are the same as in an earlier build keeps that build's narrative, so its judge prompt and verdict survive a rebuild.
+
+**Installing.** `--step install-references` takes the build, its explanations, its actions and the pinned grounding.
+
+1. `load_build` gates the build against release 20261006 before anything is written:
+ - the sidecar is release 20261006's with exactly one `engine_upgrade` revision added, from 2.15.17 to a newer engine (`upgrade_revision`);
+ - the reference CSV differs from release 20261006's exactly in that revision's changed list (`reference_value_changes`);
+ - the exclusion record is release 20261006's records, minus the regenerated ones, plus the kept ruled records, plus the records the actions newly exclude, and nothing else (`upgrade_exclusion_records`). The regenerated ruled records are exactly the spec's `regenerated_by_upgrade`;
+ - each regeneration restates the one reviewed action that asks for it, with that action's tolerance, and its target holds when re-derived from the record or the committed evidence (`regeneration_action_problems`, `regeneration_target_problems`);
+ - the traces cover every changed output, and the explanations differ from release 20261006's only on changed outputs (`explanation_changes`);
+ - the actions name the build's engine, approve only listed moves and recheck exactly the excluded outputs the revision rechecks.
+2. The audit the new references give is rendered in memory and held to `check_prompt_changes`' rule. A build that would re-open a case nothing explains is refused.
+3. The build is kept verbatim in the stage's `reference-build/`. Its three reference files go into the scoring source and the bundle, and its explanations into the bundle's annotations. `stage.json` binds each file's sha256, the engine and the actions.
+4. `prepare_audit` runs again in place. Every verdict whose prompt changes is set aside in `rejected-verdicts/` with its judge evidence, and every other verdict keeps its bytes. `prompt-changes.json` and `pending.json` are written anew.
+
+Installing the same build again changes nothing. Another build replaces the first, and `stage.json` records the one it supersedes. A stage that never runs the step is staged on the 20261006 pins.
+
+**After the install.** Every later step re-gates the installed build (`verify_installed_references`): the kept copy must be the bytes `stage.json` records, must still pass `load_build` against git and the spec, and the stage's files must be the build's.
+
+- **Judging.** A case whose reference value or explanation the build changed may change or appear without the new model (`upgraded_cases`), and it is re-judged.
+- **Adjudications.** `adjudicate-exclusions` decides the ruled outputs the build keeps excluded. It drops the decision of each release 20261006 record the build regenerated, as release 20260922c dropped scenario_045 SNAP's (#178), and records each drop in `stage.json` so that triage and the freeze hold the record to it (`drop_regenerated_adjudications`, `verify_recorded_drops`). It decides each output the build newly excludes from the spec's `upgrade_adjudications`, which must decide exactly those outputs (`upgrade_decisions`). Those decisions form their own wave, dated the revision's day, and the date conventions name the day they were written (`upgrade_adjudications_written_on`).
+- **Triage decisions.** The spec may hold `triage_adjudications`: a developer's decision on a re-opened case whose verdict flags a reference the upgrade changed. Each affirms the reference, so the output stays scored. They are read only with a build installed. This release's spec holds none.
+- **Scoring.** Every model is scored on 1,984 outputs less the build's exclusion records. The scope check puts back release 20261006's references and exclusion record together, and every incumbent's `modelStats` must then be release 20261006's (`export_on_base_references`).
+- **The freeze.** `verify_upgrade` re-gates the build and the recorded drops, and requires the export receipt to bind every kept build file. The build's references, explanations and exclusion record are frozen byte for byte (`verify_references`, `verify_explanations`). `verify_fix_module_pins` in `scripts/freeze_snapshot.py` requires each fix module the revision pins to be the committed file at its path.
+- **Judge evidence.** After the freeze, `scripts/date_haiku55_judge_verdicts.py` writes the verdict behind each committed decision on a re-opened case (`docs/haiku55/judge_verdicts_.json`) and brings `reference_audit/2026-09-28/verification/judge_verdicts.json` up to this release.
+
+## Commands
+
+Run these from this checkout, one after another. The source paths are read-only.
+
+```bash
+export OPENBLAS_NUM_THREADS=1 PYTHONPATH="$PWD" PYTHONDONTWRITEBYTECODE=1
+PB_PY="$PWD/.venv/bin/python"
+PB_RUNS=/Users/maxghenis/PolicyEngine/policybench/results/local/haiku55-runs
+PB_SEED="$PWD/results/local/release-haiku55/seed-20261006"
+PB_GROUNDING=/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/grounding.csv
+PB_STAGE="$PWD/results/local/release-haiku55/stage"
+PB_RUN=us_full_run_20260612_policyengine_4_16_1_populace
+PB_TAG=$("$PB_PY" -c 'import sys; sys.path.insert(0, "scripts"); import finish_haiku55 as d; print(d.RELEASE_TAG)')
+
+# The run is $PB_RUNS/haiku55/run: completed == total == 100, stopped_reason
+# null. Pin its files and commit docs/haiku55/input_pins.json before export
+# (done in e3fde80d):
+"$PB_PY" scripts/finish_haiku55.py --step pin-inputs --runs-root "$PB_RUNS"
+"$PB_PY" scripts/finish_haiku55.py --step prepare \
+ --runs-root "$PB_RUNS" --stage-dir "$PB_STAGE" \
+ --audit-seed "$PB_SEED" --grounding "$PB_GROUNDING"
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step install-exclusions
+```
+
+The judges bill a judge login only, never the desktop's. Run the runner directly to judge two at a time, or to name a batch with `AUDIT_ONLY`. This is the command of the stage's `launch-judges-batch1.sh`, without its log redirection:
+
+```bash
+env -i PATH="$PATH" HOME="$HOME" USER="$USER" LANG=en_US.UTF-8 \
+ CLAUDE_CONFIG_DIR="$HOME/.policybench-judge/pb-judge-config" \
+ CLAUDE_CODE_OAUTH_TOKEN="$(agent-secret get CLAUDE_CODE_OAUTH_TOKEN pb-judge)" \
+ AUDIT_ACCOUNT="claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30" \
+ AUDIT_MODEL=claude-opus-5-5 AUDIT_PARALLEL=2 AUDIT_PYTHON="$PB_PY" \
+ AUDIT_ONLY="$(cat results/local/release-haiku55/judge-batch-1.txt)" \
+ bash scripts/run_audit_claude.sh "$PB_STAGE/audit"
+```
+
+Leave `AUDIT_ONLY` out to judge every pending case, the ten ruled ones included. Then close the judging with the driver, with the same three login variables exported and `AUDIT_ONLY` unset. The driver validates every verdict, judges whatever is still pending one at a time, sets invalid and hedged verdicts aside in `rejected-verdicts/`, and repeats up to three times:
+
+```bash
+export CLAUDE_CONFIG_DIR="$HOME/.policybench-judge/pb-judge-config" \
+ CLAUDE_CODE_OAUTH_TOKEN="$(agent-secret get CLAUDE_CODE_OAUTH_TOKEN pb-judge)" \
+ AUDIT_ACCOUNT="claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30"
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step judge
+unset CLAUDE_CODE_OAUTH_TOKEN
+
+# With an engine upgrade: build, write the narratives, install, then judge the
+# cases the install re-opened (the same judge command, on the pending cases).
+# $V is the policyengine-us version, $F a build directory holding a venv with
+# that version, $U the audit directory.
+U=reference_audit/2026-10-09-engine-upgrade
+F_PY=(env PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH="$PWD" "$F/venv/bin/python")
+"${F_PY[@]}" "$U/scripts/build_references_upgrade.py" \
+ --actions "$F/final_actions.json" --out-dir "$F/build"
+"${F_PY[@]}" "$U/scripts/sweep_timing.py" sweep --engine "$V" --venv "$F/venv" \
+ --first-output "$F/pass1/computed.csv"
+ANTHROPIC_API_KEY="$(agent-secret get ANTHROPIC_API_KEY maxghenis)" \
+ "${F_PY[@]}" "$U/scripts/narratives_upgrade.py" --references "$F/build" \
+ --explanations "annotations/$PB_RUN/us_case_reference_explanations.csv" \
+ --out "$F/us_case_reference_explanations.csv" \
+ --hand-corrected "$U/hand_corrected_narratives.json"
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step install-references \
+ --built "$F/build" --explanations "$F/us_case_reference_explanations.csv" \
+ --actions "$F/final_actions.json" --grounding "$PB_GROUNDING"
+
+# Restate the adjudications of re-judged cases (writes only after checking).
+# The script is the GPT-6.1 Sol release's, unchanged; the seed is 20261006's.
+# It restates the 63 of release 20261006's 77 entries whose cases are re-opened,
+# scenario_051 among them.
+"$PB_PY" scripts/restate_gpt61sol_adjudications.py --stage-dir "$PB_STAGE" \
+ --audit-seed "$PB_SEED"
+
+# Needs adjudications_written_on in docs/haiku55/spec.json.
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step adjudicate-exclusions
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step triage
+```
+
+If triage stops, investigate `$PB_STAGE/reference-flags.csv` and `unresolved-rows.csv`.
+- A new decision may decide only a case Claude Haiku 5.5 re-opened. Record it in `$PB_STAGE/publish/$PB_RUN/annotations/us_adjudications.json`, keeping the judge's exact class, then run `triage` again.
+- The restate script restates a re-judged case's record, and `adjudicate-exclusions` writes a ruled output's. Neither is ever written by hand.
+- Published wording is corrected only through `$PB_STAGE/wording-amendments.json`.
+- No exclusion has a path in this release other than the ten and the ones an installed build's actions add. If another is needed, stop.
+
+Write the provenance record and commit it, then export and freeze:
+
+```bash
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step provenance
+"$PB_PY" scripts/finish_haiku55.py --stage-dir "$PB_STAGE" --step export
+"$PB_PY" scripts/freeze_haiku55.py --stage-dir "$PB_STAGE" --dry-run
+"$PB_PY" scripts/freeze_haiku55.py --stage-dir "$PB_STAGE"
+"$PB_PY" scripts/date_haiku55_judge_verdicts.py --stage-dir "$PB_STAGE"
+"$PB_PY" scripts/sensitivity_by_variable.py
+"$PB_PY" scripts/rescore_sensitivity_summaries.py --release "$PB_TAG"
+# With an engine upgrade, the publication check (add --check-venv, a venv
+# holding PyPI's newest release, when that is not the reference engine; check
+# runs the sweep on it):
+"${F_PY[@]}" "$U/scripts/sweep_timing.py" check --build "$F/build"
+# Update the note, paper prose, roster pins (47 models) and tests, then render
+# and freeze again, which re-pins the rendered paper:
+"$PB_PY" paper/render_paper.py
+"$PB_PY" scripts/freeze_haiku55.py --stage-dir "$PB_STAGE"
+```
+
+`export` runs triage again. It then writes `data-board47.json` and a hash-bound `release-ready.json`.
+
+The freeze is a local build and uploads nothing. Before it changes anything, `--dry-run` included, it checks:
+- the receipt, the spec, the installed exclusions and the scored counts;
+- the references and explanations;
+- the verdicts, the provenance record, the adjudications and the annotations;
+- the incumbents' predictions;
+- the payload, which it rebuilds on a scratch copy.
+
+It takes the snapshot date and the day the wave's decisions were written from the spec, and the tag from `RELEASE_TAG`. It then writes:
+- the pointer, the version label and description, the snapshot and the annotations;
+- the serving configuration, where the new row's evidence names `haiku55-runs/haiku55` (from the input pins);
+- `$PB_STAGE/predictions.csv.gz`.
+
+Then it checks what the freezer wrote (`verify_frozen`). The response window ends on the UTC day of the last recorded answer (`model_response_window`), here Claude Haiku 5.5's last one, at 2026-10-08 21:46:47 UTC. Release 20261006's window ended on 2026-09-29, so the window becomes 2026-06-12 (`MODEL_RESPONSE_START`) to 2026-10-08. The release assets are `predictions.csv.gz` and the staged payload, renamed to `dashboard-data.json`. Before uploading, check that the tag is still free.
+
+The two sensitivity scripts rescore the five runs that `scripts/sensitivity_by_variable.py` lists (Fable 5, Opus 5, Sonnet 5 and Fable 5.1 with thinking, and Claude Haiku 5.5's `tool_choice` auto run) against the frozen board's scored outputs. They must run because the scored outputs change.
+
+## What the gates enforce
+
+Each gate refuses with `SystemExit` before anything is written outside the stage. Functions are in `scripts/finish_haiku55.py` unless a row says otherwise. Rows marked (freeze) name functions in `scripts/freeze_haiku55.py`.
+
+F tests are in `tests/test_finish_haiku55.py`, Z tests in `tests/test_freeze_haiku55.py` and R tests in `tests/test_run_audit_claude.py`. The first table gives the gates of a stage without an installed build. The second gives the gates an engine upgrade adds, and says where an installed build changes a row of the first.
+
+| Gate | Where | Pinned by |
+|---|---|---|
+| The run is complete: `completed == total > 0` as integers and no `stopped_reason`. The run state names `claude-haiku-5.5`, and `predictions.csv` is present. | `discover_new_models` | F `test_an_incomplete_or_stopped_run_is_refused`, `test_the_wrong_model_in_a_run_state_is_refused`, `test_a_missing_run_artifact_is_refused` |
+| Claude Haiku 5.5's staged inputs are its pinned run. Export and the freeze read `docs/haiku55/input_pins.json` as committed at `HEAD` and refuse staged copies that differ. They re-check the prepare-time hashes in `stage.json` and `model-provenance.json`. They compare every column of its bundle rows with its run file, so its cost, tokens and latency are the run's. | `pin_inputs`, `committed_input_pins`, `verify_new_model_inputs`, `new_model_row_differences`, `export`, `freeze_haiku55.main` | F `test_pin_inputs_writes_the_finished_runs_file_hashes`, `test_the_input_pins_are_read_as_committed_at_head`, `test_the_committed_input_pins_are_the_finished_runs` (local), `test_export_writes_no_receipt_when_the_inputs_are_not_the_pinned_run`; Z `test_the_freeze_refuses_new_model_usage_edited_in_the_bundle`, `test_the_freeze_rechecks_the_prepare_time_hashes` |
+| The base is 46 models, 1,984 outputs, 64 exclusions and 1,920 scored outputs, with the pinned payload and predictions, read from git at `BASE_COMMIT`. | `resolve_base`, `base_payload_from_commit`, `verify_base_predictions`, `base_exclusion_record` | F `test_the_git_base_must_rewrap_to_the_20261006_asset`, `test_the_base_commit_holds_release_20261006`, `test_the_base_exclusion_record_is_read_from_git_and_pinned` |
+| The reference outputs, the scenarios and their two sidecars equal their 20261006 pins at every step. The exclusion record equals its 20261006 pin until `install-exclusions`. After that, export accepts in the bundle only the release's bytes. | `verify_reference_pins`, `resolve_base`, `release_exclusions_sha256`, `export` | F `test_a_reference_file_that_misses_its_pin_is_refused`, `test_export_accepts_only_the_release_exclusion_record` |
+| The ten records are the ruled ones. Each proposal file is the bytes the spec pins and holds exactly the outputs its ruling names. There are ten, no two share an output, and each is a 2026-10-06 developer record on policyengine-us 2.15.17 with a known reason code. | `spec_records` | F `test_the_spec_names_each_ruling_and_its_outputs`, `test_a_proposal_file_off_its_pin_is_refused`, `test_a_record_outside_the_ruling_is_refused`, `test_two_proposals_cannot_rule_on_one_output` |
+| The release's exclusion record is release 20261006's plus the ten. Every earlier record keeps its bytes and place, and the ten sit before the trailing scenario_023 record. No ruled output is already excluded. The derivation gains one sentence at its one marker. The loader accepts 74 records. | `build_release_exclusions`, `exclusions_text` | F `test_the_release_exclusions_are_the_base_plus_the_ten_ruled_records`, `test_the_ruled_records_come_out_in_one_order_whatever_the_files_order` (Hypothesis), `test_a_ruled_output_already_excluded_is_refused` |
+| The installed record is the release's. The first install requires the stage to hold release 20261006's record. `adjudicate-exclusions`, triage and export refuse a stage whose `stage.json` does not record the release's hash. | `install_exclusions`, `main` | F `test_install_exclusions_rebinds_the_stage_and_is_idempotent`, `test_later_steps_refuse_a_stage_without_the_installed_exclusions` |
+| Every judged verdict is bound to its own bytes. A case whose prompt is the seed's keeps the seed's verdict bytes; a failure there is refused, not re-judged. Every other verdict records the sha256 of the prompt it judged. The seed must match the committed digest. Every read of the seed re-derives kept, changed and added cases and refuses unless `prompt-changes.json` says the same. Invalid verdicts are set aside in `rejected-verdicts/`, never deleted. | `prepare_cases`, `bind_seed`, `load_seed`, `validate_verdicts`, `set_aside` | F `test_every_verdict_is_bound_to_its_own_bytes`, `test_a_carried_over_verdict_must_keep_the_seeds_bytes`, `test_prepare_refuses_a_seed_the_committed_digest_does_not_record`, `test_the_committed_seed_digest_is_the_pinned_one`, `test_the_committed_seed_digest_matches_the_real_seed` (local) |
+| Only a case that Claude Haiku 5.5 joins, or one of the nine that release 20261006 reworded, may change or appear. The nine are derived from git and must be exactly the pinned set, and each must show a changed prompt. Any other incumbent-only change is refused, and so is a seed case that disappears. The staged predictions must bear out each re-opened case. | `check_prompt_changes`, `reworded_since_seed`, `verify_reopened_by_predictions` | F `test_an_incumbent_prompt_that_changes_is_refused`, `test_a_reworded_case_may_change_without_the_new_model`, `test_the_reworded_cases_are_exactly_those_git_shows`, `test_a_reworded_case_whose_prompt_did_not_change_is_refused`, `test_the_manifest_cannot_reopen_a_case_the_predictions_do_not` |
+| The judge covers every wrong model in a case, up to all 47. A verdict that names Claude Haiku 5.5 needs hash-bound Opus 5.5 provenance and its prompt's sha256. | `validate_verdicts` | F `test_the_judge_must_cover_all_47_models_in_a_case`, `test_new_model_verdict_requires_bound_opus55_provenance` |
+| The grounding is the pinned one. | `prepare_cases` (`GROUNDING_SHA256`) | F `test_prepare_refuses_a_grounding_other_than_the_pinned_one` |
+| Each judge sees only its prompt and bills only the judge login. It runs in an empty directory outside git, with no tools, an allowlisted environment and one explicit effort. Its kept transcript must show nothing but the prompt and the structured answer. The login must not be the desktop's. | `scripts/run_audit_claude.sh` | R `test_each_judge_runs_isolated_on_the_lanes_login`, `test_the_runner_refuses_anything_but_the_lanes_own_login`, `test_every_judge_runs_at_one_explicit_effort`, `test_a_transcript_showing_more_than_the_prompt_is_rejected`, `test_account_data_anywhere_in_a_transcript_rejects_it_and_stops_the_run`, `test_a_login_that_cannot_judge_stops_the_run`, `test_audit_only_limits_the_run_to_the_named_cases` |
+| The published judge provenance describes the staged new verdicts: each re-opened case once, with its hashes, its sidecar's fields, its isolation, and a transcript that passes the runner's checks, ported to Python. The R tests compare the runner with `finish_gpt61sol.py`'s port. This driver's port is the same code today (`account_data` through `withhold_addresses`), and an F test must keep it so. | `verify_judge_provenance`, `transcript_problems`, `export` | F `test_the_record_must_list_each_staged_new_verdict_once`, `test_each_entrys_fields_must_be_its_sidecars`, `test_this_drivers_transcript_gate_is_the_gpt61sol_drivers`; R `test_the_runner_and_the_driver_judge_transcripts_alike` |
+| The decisions on the ten ruled outputs are exactly what `exclusion_entry` builds. A new decision is built from its case's bound verdict. scenario_051's is built from its 20261006 entry with the restated judge fields. The spec's classes must match each record's reason code, and the only existing decision among the ten must be the one the spec lists as restated. | `ruled_adjudication_problems`, `exclusion_entry`, `exclusion_adjudications`, `adjudicate_exclusions` | F `test_adjudicate_exclusions_appends_nine_entries_and_restates_scenario_051_in_place`, `test_triage_accepts_scenario_051_restated_then_ruled`, `test_a_ruled_entry_edited_by_hand_is_refused`, `test_adjudicate_exclusions_refuses_a_second_run`, `test_scenario_051s_ruling_moves_only_its_decision_fields`, `test_a_restated_decision_keeps_every_other_field_in_place` (Hypothesis) |
+| Outside the ten, adjudications change only where Claude Haiku 5.5 re-opened a case. The baseline is release 20261006's record, read from git. A re-opened case may rewrite its judge fields as the restate script does, and its reasoning as a listed amendment says. No entry may be dropped or reordered, and a new entry may decide only a re-opened case. | `verify_adjudication_changes`, `verify_restatements`, `stage_adjudications`, `triage` | F `test_triage_lets_a_rejudged_case_restate_its_judge_fields`, `test_triage_refuses_any_other_change_to_a_recorded_decision`; Z `test_the_gate_allows_exactly_the_judge_fields_of_reopened_cases` (Hypothesis), `test_a_dropped_decision_is_refused` |
+| The staged record's bytes are exactly its parsed content, and its note and schema are release 20261006's. Its date conventions are release 20261006's with the 2026-10-06 wave named once. | `verify_record_form`, `release_date_conventions` | F `test_the_date_conventions_name_the_2026_10_06_wave`, `test_the_wave_cannot_be_named_twice`, `test_triage_refuses_a_record_whose_bytes_hide_text` |
+| The adjudications exclude exactly the 74 outputs that the installed record excludes. | `triage`, (freeze) `verify_adjudication_record` | F `test_the_adjudications_exclude_exactly_the_installed_exclusions` |
+| Published wording changes only as `wording-amendments.json` lists, on re-opened cases, in a reasoning, a case note or one model's row annotation. | `load_amendments`, `stage_adjudications`, `amend_annotations` | F `test_triage_applies_exactly_the_listed_wording_amendments`; Z `test_a_listed_wording_amendment_is_allowed_and_nothing_else` |
+| Every model is scored on 1,910 outputs. | `build_payload`, (freeze) `verify_scored_outputs` | F `test_every_model_is_scored_on_1910_outputs` |
+| The scope check. With release 20261006's exclusion record put back, export of the staged bundle gives each of the 46 incumbents release 20261006's `modelStats` entry, byte for byte, key order included. A missing incumbent counts as drift. | `build_payload`, `incumbent_drift`, `release_20261006.export_payload` | F `test_the_scope_check_refuses_an_incumbent_change_the_ten_records_do_not_explain`, `test_incumbent_drift_is_exactly_the_changed_rows` (Hypothesis) |
+| The export roster is exactly the 46 incumbents plus Claude Haiku 5.5. | `build_payload` | F `test_the_export_roster_must_be_the_incumbents_plus_the_addition` |
+| After the freeze, a re-export reads the 20261006 payload from `BASE_COMMIT` and checks it against the base sha256. Any other pointer is refused. Once the live pointer names this release, the re-export accepts the frozen exclusion record. | `resolve_live_base`, `base_commit_blob`, `export` | F `test_a_re_export_after_the_freeze_reads_20261006_from_git`, `test_a_re_export_after_the_freeze_accepts_the_frozen_exclusion_record`, `test_any_other_pointer_is_refused` |
+| The fold keeps every incumbent row and adds exactly 1,984 rows for the new model, with the reference key set. | `fold_board`, via `prepare_inputs` | F `test_the_fold_keeps_incumbent_rows_and_adds_1984_rows_for_the_addition` (Hypothesis) |
+| The addition's treatment fingerprint matches the registry. | `validate_treatment` (in `freeze_adds0928.py`) | Z `test_the_registered_treatment_matches_the_run_and_a_drift_is_refused` |
+| (freeze) The receipt names release 20261006 as its base (tag and payload sha256). It binds the payload, the tag and 47 models; the references and the installed exclusion record; the predictions, the adjudications, both annotation CSVs and the reference explanations; the run files, `model-provenance.json`, `prompt-changes.json` and `stage.json`; the amendments when the stage has them; and every judged case's verdict, sidecar and prompt, with its transcript where one exists. Evidence that changed after export is refused. | `export` writes it; `verify_receipt` checks it | Z `test_freeze_refuses_changed_or_unbound_evidence`, `test_the_receipt_must_bind_every_judged_case_and_both_annotation_csvs`, `test_the_default_tag_is_the_driver_release_tag` |
+| (freeze) The spec names the release and the base this freeze builds: the tag, the base tag and the base sha256 the driver pins. It also gives a snapshot date and the day the wave's decisions were written. | `release_spec` | Z `test_the_freeze_refuses_a_spec_for_another_release_or_base`, `test_the_freeze_refuses_a_spec_without_its_dates` |
+| (freeze) The stage scores on the release's exclusion record. The staged record must be `build_release_exclusions`' bytes, rebuilt from git and the spec. The receipt binds them, and `stage.json` records them as `install-exclusions` wrote them. The payload lists the 74 outputs in the record's order. | `verify_installed_exclusions`, `verify_scored_outputs` | Z `test_the_freeze_refuses_a_stage_without_the_installed_exclusions`, `test_the_payload_must_list_the_releases_exclusions_in_order` |
+| (freeze) The reference outputs, the scenarios and their sidecars equal release 20261006's pins in the stage, in the committed snapshot and in the manifest. The staged exclusion record is the release's. Its committed copy and its manifest pin may be either release 20261006's or the release's, so a second freeze passes. The staged reference explanations are release 20261006's, from git. | `verify_references`, `verify_explanations` | Z `test_any_reference_value_revision_is_refused`, `test_a_second_freeze_finds_this_releases_exclusion_record`, `test_the_exclusion_record_is_release_20261006s_or_this_releases`, `test_a_reworded_explanation_is_refused` |
+| (freeze) The staged payload is what export builds from the bound bundle. `build_payload`, with its scored-count and scope checks, runs on a scratch copy of the files the receipt binds, and its bytes must equal the staged payload's. | `rebuild_payload`, `build_payload`, `payload_text` | Z `test_the_freeze_refuses_a_payload_edited_after_export`, `test_the_rebuild_reads_only_the_bytes_the_receipt_binds`, `test_payload_differences_are_empty_exactly_when_the_bytes_match` (Hypothesis) |
+| (freeze) Every verdict still passes the driver's gate against the bound seed, and the provenance record is the one export bound. The adjudication record passes the driver's gates again, against release 20261006's record in git. | `verify_verdicts`, `verify_judge_provenance`, `verify_adjudication_record` | Z `test_the_freeze_refuses_a_verdict_edited_after_export`, `test_the_freeze_refuses_a_dropped_adjudication_before_mutation`, `test_the_freeze_baseline_is_release_20261006_in_git_not_the_working_tree` |
+| (freeze) The staged row annotations and case notes are what triage builds, in every column. | `verify_annotation_amendments`, `annotation_csv_text` | Z `test_the_freeze_refuses_any_column_triage_did_not_write` |
+| (freeze) Every incumbent's prediction rows are release 20261006's, read from git. Every incumbent's serving row is release 20261006's, in every field. The new row's evidence is its run state and names the pinned run. | `freeze_haiku55.main`, `merge_serving`, `serving_problems`, `evidence_runs` | Z `test_the_freeze_reads_20261006_predictions_and_serving_from_git`, `test_an_incumbent_serving_row_that_changes_is_refused` |
+| (freeze) The response window runs from the configured start to the UTC day of the last recorded answer. The freeze derives it before it writes anything. | `freeze_snapshot.model_response_window`, via `freeze_haiku55.main` | Z `test_the_freeze_refuses_what_the_freezer_wrote_outside_the_release` (its `window` case); `tests/test_model_response_window.py` |
+| (freeze) What the freezer wrote is the release. The frozen references, exclusion record and annotations are the staged ones. The manifest differs from release 20261006's only under the listed paths, never at a pinned reference or the explanations. It states the snapshot date, 47 models, the response window, 74 exclusions, 1,910 scored outputs and the artifact. The live version's description counts the exclusions by engine. | `verify_frozen`, `manifest_problems`, `release_versions`, `verify_freezer_destinations` | Z `test_the_frozen_manifest_changes_only_where_the_release_may` (Hypothesis), `test_the_version_description_counts_the_releases_exclusions`, `test_release_versions_changes_nothing_when_applied_twice` |
+| `RELEASE_TAG` is the only place the new tag is spelled out in the driver's scripts, its tests and this note. `spec.json` repeats it, and `release_spec` requires the two to agree. | `finish_haiku55.py`, (freeze) `release_spec` | F `test_the_release_tag_is_named_in_one_place` |
+
+With an engine upgrade:
+
+| Gate | Where | Pinned by |
+|---|---|---|
+| The build's sidecar is release 20261006's with exactly one `engine_upgrade` revision added, from 2.15.17 to a newer engine. Its CSV differs from release 20261006's exactly in the revision's changed list. | `load_build`, `upgrade_revision`, `reference_value_changes` | F `test_a_build_on_release_20261006_passes_every_gate`, `test_a_sidecar_that_is_not_20261006s_plus_one_upgrade_is_refused`, `test_a_real_builders_build_passes_the_drivers_gates` |
+| Every scored move beyond $1 is approved, at the approved value within half a cent. | `load_build` | F `test_every_scored_move_beyond_a_dollar_needs_an_approval`, `test_an_approved_value_within_half_a_cent_passes` |
+| The build's exclusion record is release 20261006's, less the regenerated records, plus the kept ruled records, plus the new ones. Only an engine defect is regenerated, and the regenerated ruled records are exactly the spec's. | `upgrade_exclusion_records`, `spec_regenerated` | F `test_the_release_record_is_exactly_base_less_regenerated_plus_kept_plus_new`, `test_only_an_engine_defect_can_be_regenerated`, `test_the_build_must_regenerate_exactly_the_specs_ruled_records`, `test_the_specs_regenerated_section_names_ruled_engine_defects` |
+| A regeneration lands within its own tolerance of its audited target, and restates the one reviewed action that asks for it. The action's tolerance binds, not the revision's. | `regeneration_action_problems`, `regeneration_target_problems` | F `test_a_regeneration_lands_within_its_own_tolerance`, `test_the_actions_tolerance_binds_not_the_revisions`, `test_a_regeneration_must_restate_its_reviewed_action`, `test_a_regeneration_needs_exactly_one_action`, `test_a_regenerated_record_off_its_alternative_is_refused`, `test_module_inertness_uses_the_regenerations_tolerance` |
+| A fix module's sibling files are found from its source, and a module that loads local files any other way is refused. | `policybench/fix_module_closure.py`, `committed_dependencies` | `tests/test_fix_module_closure.py`; F `test_a_target_module_whose_closure_cannot_be_established_is_refused` |
+| An excluded output that stays excluded keeps the value it was decided on. One that moves is rechecked, with its reason. | `upgrade_exclusion_records`, `load_build` | F `test_a_rechecked_output_must_stay_excluded_at_its_value` |
+| The explanations differ from release 20261006's only on changed outputs. | `explanation_changes` | F `test_explanations_must_differ_from_20261006s_only_on_changed_outputs` |
+| The actions describe the build and are the ones it was built from. | `load_build`, `builder_claim_problems` | F `test_actions_that_do_not_describe_the_build_are_refused`, `test_actions_edited_after_the_build_are_refused` |
+| The install re-opens only the cases whose prompts change, and refuses a build that would re-open any other. It refuses a tampered build, a stage off the base and another grounding, and writes nothing. | `install_references`, `classify_prompt_changes`, `verify_install_target` | F `test_install_references_reopens_only_the_cases_whose_prompts_change`, `test_the_upgrade_allowance_admits_only_the_cases_it_changed`, `test_install_references_refuses_a_tampered_build_and_writes_nothing`, `test_install_references_refuses_a_stage_off_the_base`, `test_install_references_refuses_another_grounding` |
+| Every later step re-gates the installed build and takes the release's exclusion record from it. | `verify_installed_references`, `stage_upgrade` | F `test_later_steps_regate_the_installed_build`, `test_later_steps_take_the_releases_record_from_the_installed_build`, `test_export_holds_the_staged_references_to_the_installed_build`, `test_triage_holds_the_record_to_the_installed_upgrade` |
+| The adjudications decide the kept ruled outputs and the outputs the build newly excludes, and drop exactly the regenerated records' decisions. The spec decides exactly the new exclusions, and their wave names the day it was written. | `adjudicate_exclusions`, `upgrade_decisions`, `drop_regenerated_adjudications`, `verify_recorded_drops` | F `test_adjudicate_exclusions_decides_the_kept_and_drops_the_regenerated`, `test_adjudicate_exclusions_refuses_a_record_deciding_a_regenerated_output`, `test_adjudicate_exclusions_decides_the_outputs_the_upgrade_excludes`, `test_the_spec_must_decide_exactly_the_upgrades_new_exclusions`, `test_the_upgrades_wave_needs_its_written_day`, `test_the_adjudication_gate_accepts_exactly_the_upgrades_drops`, `test_triage_requires_the_recorded_drops` |
+| Every model is scored on 1,984 outputs less the build's exclusion records. With release 20261006's references and record put back, no incumbent's `modelStats` drifts. These replace the first table's rows on 1,910 outputs, on the 74 excluded outputs and on the scope check. | `build_payload`, `export_on_base_references` | F `test_with_an_upgrade_every_model_is_scored_on_1984_less_its_records`, `test_the_upgrade_scope_check_puts_back_the_base_references_and_record`, `test_the_upgrade_scope_check_refuses_incumbent_drift` |
+| (freeze) The build's references, explanations and exclusion record are frozen byte for byte, in place of the first table's 20261006 pins. Scenarios stay release 20261006's. A dropped decision that the stage does not record is refused before anything is written. | `verify_upgrade`, `verify_references`, `verify_explanations` | Z `test_the_freeze_refuses_a_dropped_adjudication_before_mutation`, `test_a_dropped_decision_is_refused`, `test_without_an_upgrade_optional_arguments_preserve_existing_freeze_behavior`, `test_the_mock_setup_is_the_real_builders` |
+| (freeze) Each fix module the revision pins is the committed file at its path. | `verify_fix_module_pins` (in `scripts/freeze_snapshot.py`) | `tests/test_snapshot_artifacts.py` `test_the_published_upgrades_pins_are_their_committed_bytes` |
+| The reference engine was the newest release when the sweep began, and the publication check counts scored and excluded outputs apart. | `reference_audit/2026-10-09-engine-upgrade/scripts/sweep_timing.py` | `tests/test_sweep_timing.py` |
+
+Some checks can run only on this machine, because they read the seed copy, the run directory or the main clone's grounding. These are marked (local). They should skip elsewhere, as the GPT-6.1 Sol release's do. Everywhere else, the committed seed digest and input pins stand in for them.
+
+## Invariants
+
+The tests state these as properties, alongside the example tests above.
+
+1. **Scored count.** Without an installed build, every model is scored on 1,984 − 74 = 1,910 outputs. The 74 are the base's 64 plus the ten. With one, every model is scored on 1,984 less the build's exclusion records.
+2. **Scope.** With the base's exclusion record substituted, every incumbent's `modelStats` is the base's, byte for byte. `incumbent_drift` returns exactly the incumbents whose serialized entry differs, and a missing incumbent counts as drift (Hypothesis).
+3. **Exclusion record.** `build_release_exclusions(base, spec)` keeps every base record's bytes and order, adds ten records, and ends with the base's last record. Its output does not depend on the order of records inside a proposal file (Hypothesis).
+4. **Installing twice changes nothing.** A second `install-exclusions` writes the same bytes and the same binding.
+5. **A ruled decision moves only decision fields.** For any existing entry, `exclusion_entry` keeps every other field's value and place and sets the seven decision fields. It appends the ruling's reasoning to the entry's own. It refuses an entry that cannot hold all seven (Hypothesis). It is not idempotent, so `adjudicate-exclusions` refuses a second run.
+6. **Date conventions.** `release_date_conventions` names the new wave once and refuses text that already names it.
+7. **Adjudication scope.** Outside the ten ruled outputs, the gate accepts a staged record exactly when it differs from the base's only in re-opened cases' judge fields and listed reasoning amendments (Hypothesis).
+8. **Carried verdicts.** Every kept case's verdict is the seed's, byte for byte. Every new verdict is bound to the prompt it judged.
+9. **Response window.** The window's end is the UTC date of the latest `request_completed_at` in the staged predictions, and no recorded request falls outside the window.
+10. **Installing a build twice changes nothing.** A second `install-references` of the same build writes the same bytes and the same binding, for any build (Hypothesis).
+11. **The upgraded exclusion record.** It is the base's records less the regenerated ones, plus the kept ruled records, plus the new ones, with the base's order kept and its last record last.
+12. **A regeneration lands.** A regenerated amount is within $1 of its target, and a regenerated flag equals it (Hypothesis). A regeneration's own tolerance binds where it is tighter.
+13. **Excluded set.** With any set of dropped decisions, the adjudications exclude exactly the outputs the installed record excludes (Hypothesis).
+14. **Closure.** For a module that loads siblings only by literal name, the closure is exactly those names, in source order (Hypothesis).
+15. **Publication check.** A sweep compared with itself differs nowhere (Hypothesis).
+
+## A reference flag answered during the engine-upgrade rehearsal
+
+On the 2.37.2 rehearsal, an Opus 5.5 verdict on scenario_064 Wisconsin state income
+tax (regenerated by policyengine-us #9801) flagged the reference. It hypothesized a $500
+Wisconsin limit on net capital losses, under which the $18,235 net capital loss would
+add $2,500 back to Wisconsin AGI. Claude Opus 5.5's answer applied that limit. The 2025
+Wisconsin Schedule WD limits a net capital loss to the smaller of the loss, $3,000
+($1,500 married filing separately) or Wisconsin ordinary income (line 28). policyengine-us
+applies $3,000 from 2023
+(`gov.states.wi.tax.income.additions.capital_loss.limit`), so the reference adds
+nothing back and stands.
+
+That verdict was set aside when the case's reference narrative was corrected
+(`reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.md`). The
+case's current verdict raises no flag, so the release records no decision on it. The
+spec's `triage_adjudications`, which `--step adjudicate-exclusions` applies when a
+current verdict's flag needs one, is empty.
diff --git a/docs/haiku55/input_pins.json b/docs/haiku55/input_pins.json
new file mode 100644
index 00000000..43fb635e
--- /dev/null
+++ b/docs/haiku55/input_pins.json
@@ -0,0 +1,12 @@
+{
+ "inputs": {
+ "haiku55": {
+ "run": "haiku55-runs/haiku55/run",
+ "sha256": {
+ "predictions.csv": "27a5e48f44ad875f6d0901571b91c06c67045a7a6841f95a5ab56ecf0cfc8344",
+ "run_state.json": "42f82f504003a8ec0b7604c39bb1d42d0487c48747e6124171fd86d25472bd4e"
+ }
+ }
+ },
+ "note": "The sha256 of Claude Haiku 5.5's supervised run files as the run finished, written by scripts/finish_haiku55.py --step pin-inputs from the run directory. prepare copies them into the stage's inputs//; export and the freeze refuse a staged copy that is not these bytes, and read this file as committed at HEAD."
+}
diff --git a/docs/haiku55/judge_provenance.json b/docs/haiku55/judge_provenance.json
new file mode 100644
index 00000000..3592c02a
--- /dev/null
+++ b/docs/haiku55/judge_provenance.json
@@ -0,0 +1,3025 @@
+{
+ "counts": {
+ "isolated: setup-token 2": 193,
+ "isolated: subfleet lane claude-18": 39
+ },
+ "note": "Provenance of the new Opus 5.5 judge verdicts in the Claude Haiku 5.5 stage: every case Claude Haiku 5.5 re-opened, which includes the nine cases release 20261006 reworded. Every verdict ran through scripts/run_audit_claude.sh from an empty directory outside the repo, with no tools, an allowlisted environment, no user settings and a token login, at effort xhigh; each transcript passes the runner's checks. Most ran on Max's second setup-token login (2026-09-30 opt-in); those judged after that login's weekly limit ran on subfleet lane claude-18's token. Counts are by group. Scores do not depend on judge verdicts.",
+ "verdicts": [
+ {
+ "case_id": "us__scenario_000__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:08:13.039955+00:00",
+ "prompt_sha256": "c1943e89a6fe22c99a2682b518825fccad30e4994d0ae5721e05b06643e60b05",
+ "verdict_sha256": "f1736832f26c9b0463e1b0c9401ea84592d833af117e3da1d1f2418a881b27c0"
+ },
+ {
+ "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T05:52:36.511369+00:00",
+ "prompt_sha256": "8c65b91ee004b854a0295b8aa0e7eb2b147a7679d98ec60f97e187c116204aca",
+ "verdict_sha256": "f67e7a43fee2e9f2abd7e3b8a15a1a96d5ad19f945806f1e62918acea21899c1"
+ },
+ {
+ "case_id": "us__scenario_003__head_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:08:51.570067+00:00",
+ "prompt_sha256": "72023d07ebc68ecacd9082cfffe74cf677f04e55c5e8ed763c05ae96a29acb3a",
+ "verdict_sha256": "23246851850f9d97109d8734aeb248e05175cfd13112bd29e3601b03874c48fa"
+ },
+ {
+ "case_id": "us__scenario_004__head_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:11:35.745430+00:00",
+ "prompt_sha256": "346ae03a8c5e9938a30641ab9f4d6567cb6c71c998f6a16872285a56885e5c8d",
+ "verdict_sha256": "6a0d3eea13051465988513b7ee8aaf0b1bb0cd977dcf6fc4be3d7b538f3fa4a0"
+ },
+ {
+ "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:18:27.193218+00:00",
+ "prompt_sha256": "d5158b51c6d329c0c7b8821972eea319382766ec709db21f9a9cb1f9579f7eec",
+ "verdict_sha256": "bb5560621a65bbe5ac1c3db9ef0ee746ee02c57f1ffe11cccea7b0bb26675d11"
+ },
+ {
+ "case_id": "us__scenario_005__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:19:54.244672+00:00",
+ "prompt_sha256": "45a9ad0369aac5ef287f3b960ee65816107a82cf8645a1ccdf2fd62b49739c49",
+ "verdict_sha256": "8405c7bead7b153e302ab89b0eca5be588ef09e3679f1b961718d708bdcf657c"
+ },
+ {
+ "case_id": "us__scenario_005__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T05:55:57.066505+00:00",
+ "prompt_sha256": "23996f96494e704f4290f3866ccd8dd7b43ef41b528a296c43325a7d5c5a61f6",
+ "verdict_sha256": "7653eda025bcf5be81805f01e4374537d2322c914c1c066a1dfb76445c902059"
+ },
+ {
+ "case_id": "us__scenario_007__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:24:05.267634+00:00",
+ "prompt_sha256": "8830d4d963659b210068f5b4bd6b82e19a132dea099f4a457c4c61b65dbec59f",
+ "verdict_sha256": "945362b7e820ec8b6f606a63583bd85f7e7683b46e94ead1e3a4cd913e54b6d4"
+ },
+ {
+ "case_id": "us__scenario_007__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:01:12.293170+00:00",
+ "prompt_sha256": "bbce3b38ab1b15be53dae2e810c50bfed6ab82415efdab80c27c7b0995efdf60",
+ "verdict_sha256": "ee385050ee1a01f48f66e1d6160621e1867c15de9faac56439cf0db3634459c6"
+ },
+ {
+ "case_id": "us__scenario_007__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:25:24.334146+00:00",
+ "prompt_sha256": "b70bdc21cbef932867199513bf1f249c9c1b1d9bcd29c36dc8627baaacd4842a",
+ "verdict_sha256": "f08a59b24d7dc4b1ae4ff60d1a26dede646b93fcf4a012f76ff27b04b47316e7"
+ },
+ {
+ "case_id": "us__scenario_008__child5_wic_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:24:57.601322+00:00",
+ "prompt_sha256": "384d92fb4ddf152e224f01812c72f6040101fdc9893674450a035913387aef3b",
+ "verdict_sha256": "b821cd6e9f5d1675d866f37b70025149cae771fb1b89a10572c7345f9f365e35"
+ },
+ {
+ "case_id": "us__scenario_008__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:26:51.398295+00:00",
+ "prompt_sha256": "4e142d9c9f92b32971477c918799d6bc574faf08fcd00c728ac211f81c9dec6d",
+ "verdict_sha256": "519e5dc051a016eee4a14bd9aa57b953d04f39c403b8d833b147a6f6eecaba37"
+ },
+ {
+ "case_id": "us__scenario_008__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:25:45.498744+00:00",
+ "prompt_sha256": "8a1564f0c291cabb2b87780aef66a5832940e45e865c7b56b5878d5de307ca33",
+ "verdict_sha256": "b61da31bf48c969ac3c242f02853a1d3060f8643084090da2e3ff23c60d30013"
+ },
+ {
+ "case_id": "us__scenario_008__payroll_tax",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:35:53.817597+00:00",
+ "prompt_sha256": "626c33c358a9c2ea5c1809fa5c7629c3625f5a7be799aa9ebc879c0bd04616b3",
+ "verdict_sha256": "9ca4a0876e65a9060128f3c9408752059315ba991f182d9aeaa705619ffaf2f4"
+ },
+ {
+ "case_id": "us__scenario_008__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:28:24.173899+00:00",
+ "prompt_sha256": "62f0d8ec99b7362e549c8c146c86b993112a98fc8c59aac859615341e7011bab",
+ "verdict_sha256": "8ac11fab6007469a226c8eab6a10cc296b0d409c93fbb63501c9eb65bb164c27"
+ },
+ {
+ "case_id": "us__scenario_008__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:29:45.774860+00:00",
+ "prompt_sha256": "beeceb686e608a23900762839027ee423a6e2df8a9389e05dd8d0c88f8d76139",
+ "verdict_sha256": "c05aafb2f3e54e2037fa3f939c06d290463b53f639c63766fbb27284971732d7"
+ },
+ {
+ "case_id": "us__scenario_008__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:30:24.349878+00:00",
+ "prompt_sha256": "e5444d0a567e1f4b8d622a8bb5c94f4d5c593c11716b024e6fe23d041f11f6a7",
+ "verdict_sha256": "bf20e47d5085be40b1e41050d951b428c86fe531dcb1e469a366f7e0e2dce003"
+ },
+ {
+ "case_id": "us__scenario_009__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:31:36.427551+00:00",
+ "prompt_sha256": "243c9067e20a820bbdcafac39abeabca3fa6ece4d3316044567d7878a8989573",
+ "verdict_sha256": "6420da89a974cdefe8b8f652012ce92c2672e01ac659c0bf7aba223b9f96d60a"
+ },
+ {
+ "case_id": "us__scenario_009__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:30:52.136839+00:00",
+ "prompt_sha256": "34260f5eb3599bba24f24bf7c496901b1e00804a71b3b0aa310f2d1cd212584f",
+ "verdict_sha256": "2a9b2ae1dc24cdd9af7489f87c4aa055a417e39d97cdaca05818e6afe6b39565"
+ },
+ {
+ "case_id": "us__scenario_009__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:32:24.006780+00:00",
+ "prompt_sha256": "d7c24b29233ba300b0590c1ee58007f1cee67d4b0a401864d3806e8d859521d4",
+ "verdict_sha256": "d8be15d3cebc5ffa3f6917dbfde74c40b6be5efae67ad236ff761a9400058e37"
+ },
+ {
+ "case_id": "us__scenario_012__child1_chip_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:32:01.636134+00:00",
+ "prompt_sha256": "5f78a64edca0abfdef6b3607253f3c8c960614d88a5b811eba5feaef25c4ffbd",
+ "verdict_sha256": "20edc876c9e6010755cff47f8d4e76c021847cef4dfc9289572ea39fc56e9a1a"
+ },
+ {
+ "case_id": "us__scenario_012__child1_head_start_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:32:34.163129+00:00",
+ "prompt_sha256": "90dddff7843a5d2fde3f441347ff088c59c72968d1a9c24fa4fb6b1f204cdb0c",
+ "verdict_sha256": "dc2210c198d414aaa124f83a59c60d021270c4f8caae3b42949deac40a240a41"
+ },
+ {
+ "case_id": "us__scenario_012__child1_wic_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:32:36.459342+00:00",
+ "prompt_sha256": "e77cd5f13da5e8a407794921a9534f7ca6af6581dbe522575dc9b343313c85d7",
+ "verdict_sha256": "88629e7d905733a3ff48ae1b30fba4f2cfc067f7489b052e8df5eadf80ccfe5a"
+ },
+ {
+ "case_id": "us__scenario_012__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:33:42.931106+00:00",
+ "prompt_sha256": "7740d513f8593cb2f0a8e254d345e4581089d545c46d79ff127c8dd619f84571",
+ "verdict_sha256": "5fc3c8de1d6d6945fc24f545af9d9001802ef65dfc5e3b43b128e42c6957022f"
+ },
+ {
+ "case_id": "us__scenario_012__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:33:59.296064+00:00",
+ "prompt_sha256": "32780900c4c47525cf9935c1e0af0dc2a0960992dc4f3a5cd1aa281a7c4e94d1",
+ "verdict_sha256": "adbca0f364c387b0bd7a9f70f08a35df6e71b1827e914fb6f01e824700a76c22"
+ },
+ {
+ "case_id": "us__scenario_013__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:35:27.738658+00:00",
+ "prompt_sha256": "9df47c565cfddadba7768500ce9d00aab65a603c8067724499d8e24405629668",
+ "verdict_sha256": "fcbd4e34d6accb57de06df7a25432dae8c8edfd25c2c3ec52ea2aa9f18ebd26a"
+ },
+ {
+ "case_id": "us__scenario_013__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:34:56.548996+00:00",
+ "prompt_sha256": "838626f1d3518944b57f2b1efa869d0bb51d0c655f5972220bad6dcd3a0b8d4e",
+ "verdict_sha256": "4c6d60213e9ad24fe5f043da7bfd5e98078d355bc223488adc905503f93f8e16"
+ },
+ {
+ "case_id": "us__scenario_014__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:36:33.112141+00:00",
+ "prompt_sha256": "264ec8f272456c70e038bcd5c1cf16bc0b07ead1276471ed3a2f0ea7beb1d214",
+ "verdict_sha256": "b7519c60dd34d82acb99e8554d55a5f798a8b18435508bf9a8b08ff317925bca"
+ },
+ {
+ "case_id": "us__scenario_014__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:37:05.913752+00:00",
+ "prompt_sha256": "80fda80baa74d25c21d45fa438f9b3a5bd0008be4cc7fff8b1af0055edcb47f0",
+ "verdict_sha256": "2f3561012e32038914ddcfa2ad4d18501c361fae58b338a4669edfe9e67c9c60"
+ },
+ {
+ "case_id": "us__scenario_015__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:38:32.026841+00:00",
+ "prompt_sha256": "699a87518882b4e6556f2bb230f5abe18f8fad549a257ce7612a07c6e19315f8",
+ "verdict_sha256": "f4b21ebd85c8bbb6ee29eb6ee7004db26fad2dd289bdde6f6d5a2d067b1bbf7e"
+ },
+ {
+ "case_id": "us__scenario_015__local_income_tax",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T05:58:15.427599+00:00",
+ "prompt_sha256": "00bf68e6fdd682fa92e1eb11d4777e3b7fd1a3790bd888f61c67d2af9a9c199f",
+ "verdict_sha256": "e2f324f633297e6cfcd3832de8cba9d5a71b8f022d3bb0694dd30a0cb6e9ce91"
+ },
+ {
+ "case_id": "us__scenario_015__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:38:24.813806+00:00",
+ "prompt_sha256": "a6e0ae7db6a34ce88caed3aaa8eb27526642c281af513cac9109f5c69e532741",
+ "verdict_sha256": "16ef6e0393563311831d4593e17dc020318ca3291aec6112fa886a44bd3cae42"
+ },
+ {
+ "case_id": "us__scenario_018__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:40:24.744570+00:00",
+ "prompt_sha256": "e1f69393d34e79c43ec690c3899f07bfd7692b372c962b1ee49651722f091c3b",
+ "verdict_sha256": "2e31dc076b6f4cf682f88bbc9860ddcff3b37f9552d3ac4d51fea7b48d4488f3"
+ },
+ {
+ "case_id": "us__scenario_018__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:32:52.788727+00:00",
+ "prompt_sha256": "d5f7f7e6ab5051bfdc5df27a613778e29adfa743d628b5734bdef3488a634e84",
+ "verdict_sha256": "6a2f8b3a2424f0eecb3af8a0fa6fffe2947c3102d7f7317fb1d7e14b303f8b23"
+ },
+ {
+ "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:40:06.525009+00:00",
+ "prompt_sha256": "3d56d4656a2989443d286d4482f8e9539ed381621e968492b0a7a227beb0d128",
+ "verdict_sha256": "154da3be56f8792356ad68c1df2e687bdce6098354aa78c42b55df707222504a"
+ },
+ {
+ "case_id": "us__scenario_020__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:41:32.599380+00:00",
+ "prompt_sha256": "b572076e3330493639c7c9bcf127beae6c01fec3e70d0f923e451aefa3ded630",
+ "verdict_sha256": "7ddc85df8e77c6d5ac46b2c1a720c057a447419d599f5b868e68c4e75cfb65b5"
+ },
+ {
+ "case_id": "us__scenario_021__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:40:57.588981+00:00",
+ "prompt_sha256": "0489d1e0286f1a0810e20b940add7243f7330750ea8d85eeeddc6db301fb41ae",
+ "verdict_sha256": "d340d4920b69e4f5ae8e4dbbf165e39677da72244115266110c91844236abcaf"
+ },
+ {
+ "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:44:46.816224+00:00",
+ "prompt_sha256": "8150bedc96fc372abc3aecd3e4dad0a8e8bddb16e52dcde7212af3f56d001515",
+ "verdict_sha256": "7beb1c74f6cf134e461424cb7adb52ff8d2c0248268316317bc8c1f2a3e9ea9f"
+ },
+ {
+ "case_id": "us__scenario_022__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:42:23.754548+00:00",
+ "prompt_sha256": "b6c1b2c9c0d54874e6e9aa7bc222a8bdf9afb87c009c4f531b8fe4b8599a606d",
+ "verdict_sha256": "9761e01dbb3206827079781a3e369b1bbf7b33325d7792884ad7f36ae93bbebd"
+ },
+ {
+ "case_id": "us__scenario_022__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:00:15.983475+00:00",
+ "prompt_sha256": "39d7184ce67cc967e4f96ae49fb90ab607c14e0715aab5364233b19c2bed6f8b",
+ "verdict_sha256": "e53852d0b72d09bd0af25189a039828bf69b14d3bf3e25d33656995c9caa5316"
+ },
+ {
+ "case_id": "us__scenario_023__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:46:13.366135+00:00",
+ "prompt_sha256": "fccb23c6f0195491583ee73635d63bcb209ae7a0e0d73272115180a3691fedb4",
+ "verdict_sha256": "68703a97a7f9ecf4a0d1eb64dd7ba9c252cc9cb0e833d204f108846a70278321"
+ },
+ {
+ "case_id": "us__scenario_023__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:49:07.881342+00:00",
+ "prompt_sha256": "190659f237d1dd022a966cf85802b937923c5be4fbc80f3d5a65cad7fd8927a7",
+ "verdict_sha256": "d1263a6c3cfb4d629b05ed061dd0c7f9d1ec173c4951232fd0f7339d5b3745cf"
+ },
+ {
+ "case_id": "us__scenario_023__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:49:01.373262+00:00",
+ "prompt_sha256": "38df4bbfedcbb2c96f82d22785248a55788749232e98c350c2aa975920b57eae",
+ "verdict_sha256": "6fbdba12726aa2d327d1fa0cb1d1a109df45ec606d008445afd24d7eb8d4e216"
+ },
+ {
+ "case_id": "us__scenario_023__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:50:54.620139+00:00",
+ "prompt_sha256": "66d7c9331499cff86a3e54cfb8e8a17332ba0be02bce66f5b05d9baaff9c2034",
+ "verdict_sha256": "cbd9abdb5318469c30d4aa62f2cf3dce7a5dc43d13b67553c9e3e515f5aa90c1"
+ },
+ {
+ "case_id": "us__scenario_023__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:05:39.494559+00:00",
+ "prompt_sha256": "fe8c98e1d62dbb5bc955c933c77ec4210a9d8975aa54733a3e9cd232552105c6",
+ "verdict_sha256": "de4c2be8725adc9b4f51d4d882d77737372875c9b323f7038260d66fea69a87e"
+ },
+ {
+ "case_id": "us__scenario_023__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:53:10.817427+00:00",
+ "prompt_sha256": "434f2e6d5457bf852dc5cf9ce4d85af3b8b03d173b1356d9459dfa9a6b0340ea",
+ "verdict_sha256": "f1a357ef4a6b1adfb2f1ee07be7019147feb51a0fbbc5c027b0262cd42297d15"
+ },
+ {
+ "case_id": "us__scenario_025__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:51:47.899644+00:00",
+ "prompt_sha256": "d9e73b237259924b17207bed8b112271109939e8def3b9543ff88eb71f24bc3b",
+ "verdict_sha256": "e9368fc9ac5ba87f14cd81f4810c4ffdc7b89cb10b86564b20a39cfb72ba94d6"
+ },
+ {
+ "case_id": "us__scenario_025__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:53:42.327132+00:00",
+ "prompt_sha256": "e745166fc15851cda53e9847d63a153aada489bd6281212505fabd8d8b8b8d77",
+ "verdict_sha256": "c5f274c05ca37d4762a3c18d2e5233214b4b93f8b355480b94e7ea480e14ccc3"
+ },
+ {
+ "case_id": "us__scenario_025__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:34:40.933531+00:00",
+ "prompt_sha256": "3c4b71b102f038ff3c9d164e0795d1f108ac0250c783458de52d06980a6d2c33",
+ "verdict_sha256": "4bbe08e82d808c0817e5c8f5048e0101441f63bd6492fefceabe73cb34c02fe9"
+ },
+ {
+ "case_id": "us__scenario_026__child1_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:54:36.647232+00:00",
+ "prompt_sha256": "e73080fb927f4f8fbe169491fd055ea223b49a88a9c0ee2e3f2cb5162eb88473",
+ "verdict_sha256": "94f16fbc73048676eeec5238e1b14f8be8bbe065f999319d54bffdeb5f9b05cd"
+ },
+ {
+ "case_id": "us__scenario_026__child2_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:55:58.866578+00:00",
+ "prompt_sha256": "9c984b293482ecf03b4c7084746c9b025bd59dfe405e66484793dcc3d09c279e",
+ "verdict_sha256": "3c34a95ffbea534a26058d49ce7f05cc691e12c9be489e58c5768c86da88f07d"
+ },
+ {
+ "case_id": "us__scenario_026__child3_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:56:03.245241+00:00",
+ "prompt_sha256": "497087c1a97ea358dd7f44b2dd0bb8d111a12f931cad4c5c72adf2c672a80e1a",
+ "verdict_sha256": "f211d327e48f01fc7ec3ea31cc2a76d39c46ba74a463428a6cc9efa7598fbcf0"
+ },
+ {
+ "case_id": "us__scenario_026__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:57:41.439188+00:00",
+ "prompt_sha256": "73300f41a742420ec319d8feb2e0ad5e541c7186a7cd1e21fe8f4c67e8428549",
+ "verdict_sha256": "e428c0318924513176b6638a9ed00bb7fcfdffc564482f711d7aaf9090ceea39"
+ },
+ {
+ "case_id": "us__scenario_026__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:56:30.485443+00:00",
+ "prompt_sha256": "40a7045cc13f30876297c0f3a86c8f476c96f55c5c4131243fe42af54ed43149",
+ "verdict_sha256": "29363ab09f913fa9f50a6d4dbe134cd4e2edd8d4f5a0b0ab13f85bbbab28c2e0"
+ },
+ {
+ "case_id": "us__scenario_026__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:58:51.558609+00:00",
+ "prompt_sha256": "987887e41b1a4987bb011ee399f447e5aaaed7cab48acde16ab5954c5518aff6",
+ "verdict_sha256": "fb0f952f37680280f30fff8c21016479b68bc28806c48dd89f1bbc6f7cf7f850"
+ },
+ {
+ "case_id": "us__scenario_027__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:59:04.654023+00:00",
+ "prompt_sha256": "a03e0f1722da33adaad17d9171b40c6832855c5962e23c2d5d86612d99c076d6",
+ "verdict_sha256": "97dabd25fc0a852d8ee88a23fb2030af6fc4e601f5805b391eecafce1f47affb"
+ },
+ {
+ "case_id": "us__scenario_027__spouse_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:59:26.124336+00:00",
+ "prompt_sha256": "2c1891cc3b9ce6592806b394125c16e3c792113360ea4f76e53f915ab0b1ae7f",
+ "verdict_sha256": "e51cdbece42523623127b0d4e060891d12ac0b0f1e69e2f6b6893573492ca2ff"
+ },
+ {
+ "case_id": "us__scenario_028__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T22:59:47.465843+00:00",
+ "prompt_sha256": "9a5de242dd6c7bf8259c07aaab1c89c062a9354776c7240844a0b83abef923bb",
+ "verdict_sha256": "e636232f8ccc9b3bda2be5bc1b0f1d424247db0b239a02bfe1c407b74e980149"
+ },
+ {
+ "case_id": "us__scenario_028__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:01:13.810634+00:00",
+ "prompt_sha256": "b7250abdad7765cd103918677bad76f8e36138968b2c3d2ed87750e84af43525",
+ "verdict_sha256": "703f153771007ee495217db0a2b7d9ed41ee0ba9b0d4bc1919a8b784606eea5f"
+ },
+ {
+ "case_id": "us__scenario_028__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:00:25.293012+00:00",
+ "prompt_sha256": "50e43cbb3711731d858f6f929c1b37d701de64edd47aa183e6574e16edbf8371",
+ "verdict_sha256": "09c1dd3371add125f78ef9586396b5b6bcb45120cc27d3d182d1a9f291182f65"
+ },
+ {
+ "case_id": "us__scenario_028__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:01:52.039442+00:00",
+ "prompt_sha256": "967b05135d442fbb84bba54cd1c30acdd5f5942525f9c8f4c78b79d3d4ab1901",
+ "verdict_sha256": "4080dcb2782f207dbd5b93bc39f4aba03142ed9c22c9480c1a134941c3a1583e"
+ },
+ {
+ "case_id": "us__scenario_028__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:02:03.172995+00:00",
+ "prompt_sha256": "2f6c81ded6bf2b951bc08f32e1def26c47899e08706bf6c95c150633fd7ac6e0",
+ "verdict_sha256": "17845b6db0826010f994abc3e4cbb015977cff448f1083d33d4ea1826840d741"
+ },
+ {
+ "case_id": "us__scenario_029__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:02:37.369635+00:00",
+ "prompt_sha256": "425282aaef2058fd92b24d6002a29dd7ee40cf66f7fbcfa0f2449979b42fca15",
+ "verdict_sha256": "13274b58aa487ed686c5539598e2733f0412f27a4b45bda02556ecafb8f3af82"
+ },
+ {
+ "case_id": "us__scenario_029__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:02:58.587865+00:00",
+ "prompt_sha256": "5fac6b0cd57c94d3a3685aa5222303a3c1475de07f1885173b0d6e1eb8980503",
+ "verdict_sha256": "2cc0760d15f7d536a52b5a75ddcb0c5188e4188f56b185b9dc29464c88de1326"
+ },
+ {
+ "case_id": "us__scenario_030__snap",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:08:47.548119+00:00",
+ "prompt_sha256": "3f7597cccb44f2494c1af740b867b917ed240380e7c9c7c9d2d3103141126871",
+ "verdict_sha256": "8eb4fa2cba889e3d8b0516646fc53a581af94b342f06ba49e6240e099bdbc18b"
+ },
+ {
+ "case_id": "us__scenario_031__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:04:01.743190+00:00",
+ "prompt_sha256": "038f4a0ade9f0f1d3b7f3d693720d9338d536fd33ac9548a0b549cc12cbbd11d",
+ "verdict_sha256": "50e93ea238f2cda312559468aeb3e80267e308733c838a4d31c09903cc4602e3"
+ },
+ {
+ "case_id": "us__scenario_032__child1_head_start_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:04:41.820450+00:00",
+ "prompt_sha256": "d0693813aa4587fee63f6a5f0a86ccd7c7c9896ba67e720ac179503a7b85d3ab",
+ "verdict_sha256": "0ce86babafa6a95898b1bd421b6192323efc8b875af4894401eb31eed30c2a6c"
+ },
+ {
+ "case_id": "us__scenario_032__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:05:25.717730+00:00",
+ "prompt_sha256": "c692ff7cb52bd2df79dd75099a3966dbb46a12a7b3ad9684d10d6d8d708f5bbd",
+ "verdict_sha256": "88cd884ba421dace2249090a7948cf1a729696a2c912f1521cf6adb222a362b5"
+ },
+ {
+ "case_id": "us__scenario_032__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:07:05.362213+00:00",
+ "prompt_sha256": "14a645942f03a815209b9534763fa10822c957e1b2619dbee41c26ca6abc7844",
+ "verdict_sha256": "bf538655ee75fbab32127c3d2edcc64b8ddfc3be81c00e919afb5b2a1e8e01bb"
+ },
+ {
+ "case_id": "us__scenario_032__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:07:35.419088+00:00",
+ "prompt_sha256": "858e34402d4b4b473d7843308f4b2fddfacc8d97c853032cd14a4bd4570908c1",
+ "verdict_sha256": "a73d2e58cde17a3428ec253f40e7c3422502a8b20a5edf4975cd97b653a399db"
+ },
+ {
+ "case_id": "us__scenario_033__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:09:02.707654+00:00",
+ "prompt_sha256": "d5dcf9d78866c9b78b87b9e38c76cc41476ecaa1e2479b9eedff46e2fee0ea75",
+ "verdict_sha256": "6337684e8c8766368358f1f37b49ba71a0915cf10b9b3183b60bf5882fc8be0c"
+ },
+ {
+ "case_id": "us__scenario_033__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:07:58.346050+00:00",
+ "prompt_sha256": "c9aa4c2d6fcb1c429d724297793a2a5fb209b50cca76c22b9f5d7bc408cda0bd",
+ "verdict_sha256": "5598d0c0d0f73374a2a854fa23c96e58fc36ab3b6c6d1133c3d5028ff15b50ef"
+ },
+ {
+ "case_id": "us__scenario_036__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:09:56.479259+00:00",
+ "prompt_sha256": "12e2f8b805dc8403d6f527af6b8218f4de726ede07e1becfbcde2b7d77d008ae",
+ "verdict_sha256": "c635aecbf7b10f8d6acd2d83abc4b6db93d76c9bdbe55f6b8c19a3be245eac83"
+ },
+ {
+ "case_id": "us__scenario_036__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:10:28.074849+00:00",
+ "prompt_sha256": "0111a30f4716105d3be653c00a765631f7900881277474802d9a9d14d537bfc7",
+ "verdict_sha256": "e115b451376863639eee6497ef7c9482fb218c22db38315d72670046f2779ff7"
+ },
+ {
+ "case_id": "us__scenario_037__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:11:38.163916+00:00",
+ "prompt_sha256": "34b95bae28bcc8829cd1cc183210c49c2076006722dc47a774f83deb78ff5dfd",
+ "verdict_sha256": "65172a29c6d7e5dbc7765632afa4fefe6ec5d5d3a8191c6be8d2b8f53768d9aa"
+ },
+ {
+ "case_id": "us__scenario_037__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:11:03.788235+00:00",
+ "prompt_sha256": "794845071e5569220fe34637c8d32b60e72a547a755adcfab63277f96dc1ec2e",
+ "verdict_sha256": "06792bb759b9e082788825aab174be85f3a4ad49d16a83a2fbd915e52ffa3bd0"
+ },
+ {
+ "case_id": "us__scenario_037__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:12:57.494247+00:00",
+ "prompt_sha256": "f8a5aeb255a33c077e71cc735833b1944ae1d4a7b5d95d645fb95d9294c2d208",
+ "verdict_sha256": "54b9e7cc8feb488238c99d13b0bdce37be112e9dc9c5520b11caca1b8d40abdf"
+ },
+ {
+ "case_id": "us__scenario_038__child1_chip_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:11:58.452939+00:00",
+ "prompt_sha256": "e1925f16f7e0f00d7d6ce5df76e316cd6132c22d9d6a72190cfe69688211dc89",
+ "verdict_sha256": "ca207c7a8003b639fdaef9d7f5a9544995e698fa44281e84af7e2155cc22e2bc"
+ },
+ {
+ "case_id": "us__scenario_038__child2_chip_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:13:15.052279+00:00",
+ "prompt_sha256": "19b2f317f8aae78a6082166170b20059be8a377b88d9a20c410e0383fe398b5a",
+ "verdict_sha256": "b1ece33f65e3c872c1daf8d39d864fdf9d3c312f9d45b26da86449cf10aea184"
+ },
+ {
+ "case_id": "us__scenario_038__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:14:20.506699+00:00",
+ "prompt_sha256": "8440a3f49bb7ef5ffb70c82756e36a5f07b7e606212e8634961e49393e5eb402",
+ "verdict_sha256": "efae871ea4254767e578148d8f31d42afad5749c5bec08d045c99a18f5947367"
+ },
+ {
+ "case_id": "us__scenario_038__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:14:33.244263+00:00",
+ "prompt_sha256": "04b06b5787ccbd8b67b38d9731e78bb56302bd7659c0332d8b8d754ac8a0d4f8",
+ "verdict_sha256": "0f9133393401e180cb8ed18309472ca6075dd20f575f006a54f89b7e6aa247df"
+ },
+ {
+ "case_id": "us__scenario_038__self_employment_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:14:42.953566+00:00",
+ "prompt_sha256": "e0effbfa9c71fffc7b672bf908206adea97d94bb4ae8636efc1e22ab3f6d4e73",
+ "verdict_sha256": "272e45fa5dd4044c8c105defea5f4521461d6af8199eb9105d8c1810d3a82bc8"
+ },
+ {
+ "case_id": "us__scenario_038__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:15:55.631479+00:00",
+ "prompt_sha256": "827794e6727ec6439ac49d346cd4e6090ccea7c1ac15349062223f2befb553af",
+ "verdict_sha256": "baf8d558384a62ff9fa5fde71a1da0a7418d972391aa7b2a8abd70f210d1ab13"
+ },
+ {
+ "case_id": "us__scenario_038__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:15:36.682746+00:00",
+ "prompt_sha256": "09e72b96f1ba4e12a2a6b605e26ffa6662f6bcb4034f3f72ee80d6731773e2eb",
+ "verdict_sha256": "9785a81a921ea902039cfd3f1facec4f0b1ebeb4b84accac56b97c95da5b7df7"
+ },
+ {
+ "case_id": "us__scenario_039__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:02:02.624589+00:00",
+ "prompt_sha256": "91056adf52e6a7fbc994dd1d0f0f9ca0db9c2d9d0fcf11b05b365d93205df487",
+ "verdict_sha256": "eccc9b71762cf4c9edfc4cd7c618b5dd0f544d7d4c7a8a69e9a9799b133cc860"
+ },
+ {
+ "case_id": "us__scenario_039__head_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:16:37.609603+00:00",
+ "prompt_sha256": "ebb802b330c78600500d62e4a6d7d1ba0d56323c5dba5c52a80f076f341d9a20",
+ "verdict_sha256": "a06d875a4465d0eef26e9c1cc1f8c31411c0213f3ac4cf16b0488f95314fb1dd"
+ },
+ {
+ "case_id": "us__scenario_039__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:35:59.628502+00:00",
+ "prompt_sha256": "2c1730aa22455590cf03526bfbed9a6a0c2a33f2cc903dd5205adc5761f6ef62",
+ "verdict_sha256": "0cd1b0396319c8b5a12847afc235374257820dc3047bc1d043db973483b5c867"
+ },
+ {
+ "case_id": "us__scenario_040__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:20:18.978868+00:00",
+ "prompt_sha256": "1fed60f4dbe4e65da843db4b0bdb738697d29ee8a839e758bf0746b12b05f140",
+ "verdict_sha256": "13093a99dfbc9ff0c9edcdaca52ecd6e74ee88e7f2472e1c65a45bb1c2d46775"
+ },
+ {
+ "case_id": "us__scenario_040__spouse_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:24:13.934692+00:00",
+ "prompt_sha256": "679d096fcf26e8174f6ceb7414d5a547aa6ab2dd2c9bc7c3612c3654b2573417",
+ "verdict_sha256": "0ab3d78dc2fff266c44f24738f400dd76fd110b5c2de5be0b49465d435a1f888"
+ },
+ {
+ "case_id": "us__scenario_042__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:25:33.269671+00:00",
+ "prompt_sha256": "ad7c9420ce79d3676f6fae0a72c6b2cf85afcb5ee938973619011178994da199",
+ "verdict_sha256": "780085ada860c36c43125ac03a11dfd43557032cdf8c505d0b4f3c705e5e9676"
+ },
+ {
+ "case_id": "us__scenario_042__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:31:55.110227+00:00",
+ "prompt_sha256": "6260c9b25f7b4c0cd3996ce2b0c40b7bd8dc33fcc627b51715bbdab7e2e9a9c9",
+ "verdict_sha256": "68eed26f001c269c63b9a788f584a673bc9b70328a0052f3e62dc477df55e8f4"
+ },
+ {
+ "case_id": "us__scenario_043__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:26:25.548985+00:00",
+ "prompt_sha256": "20880542a485c03ab864cabc8c6f911edbec932509d46f3f0d3576a877ded70d",
+ "verdict_sha256": "7a0dda0e9550e268300a5d61435749293eb7e02a7292b1656c7cb28b0efa7340"
+ },
+ {
+ "case_id": "us__scenario_043__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:32:56.858485+00:00",
+ "prompt_sha256": "3a7435cd12a0c06608f916f999f18caa38b75193e97cd28f540bf5e1e33ffdd9",
+ "verdict_sha256": "adc8d2a7f27c6c5fb773f5beb555d14dbad7bf7c86a8befdb37ca024ae74ade5"
+ },
+ {
+ "case_id": "us__scenario_043__state_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:36:41.899985+00:00",
+ "prompt_sha256": "5c453100bf56dbb03a7662fb321ddfbf2ceaaf4e1b1d12b77840c5ea3731774d",
+ "verdict_sha256": "e06651bfa881e21214841a6e66a01ab869c72e2edf245d51613e28ed9a236996"
+ },
+ {
+ "case_id": "us__scenario_045__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:33:02.748480+00:00",
+ "prompt_sha256": "c9a335e6710514f6914b76f0a18f9ef0852dd5e60175516bdfb1141219761950",
+ "verdict_sha256": "c42e06789c733ac8c469875e79ab4e01bd326c709157db02f50730aa0b9fc638"
+ },
+ {
+ "case_id": "us__scenario_045__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:34:01.822227+00:00",
+ "prompt_sha256": "fb4ddd500fa197f8d137e369dca30f866ff6c39e00c698bcf809b99b061c97bb",
+ "verdict_sha256": "5930b0ab8846d8083ea8d718bf0cf016b880b762c6c205a70653611fe1b85281"
+ },
+ {
+ "case_id": "us__scenario_045__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:34:12.170835+00:00",
+ "prompt_sha256": "864ef2fc785cc521ed3c07310a60b1c161f332857226f3ff6397ac4a19ce5ebd",
+ "verdict_sha256": "54557967511412c9833f205ef8754fce4f3942b65f12d7b059f50ea213d273ef"
+ },
+ {
+ "case_id": "us__scenario_046__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:35:30.695642+00:00",
+ "prompt_sha256": "5fd995e4e5bc0cc2a5e5e97b90769f4a24f31bc1638a83a46103bf547022d49d",
+ "verdict_sha256": "a197f268d2359139996d43ac45b84c416cdfae7188c4d3e1cb82901800f32404"
+ },
+ {
+ "case_id": "us__scenario_046__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:34:48.969808+00:00",
+ "prompt_sha256": "1e60d329f42a14c9c655c42f96c11ee11b7a2dc70d6f1f0fc6e34c8a7d2edef9",
+ "verdict_sha256": "fff22ccf33680808bd5beda05a5caae3e8548dd0d886a282539998f7009d7932"
+ },
+ {
+ "case_id": "us__scenario_046__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:38:06.427491+00:00",
+ "prompt_sha256": "fb180d71e3e5468299518a204824aaae3f770c6eee51d01c85345811cd08333c",
+ "verdict_sha256": "5ab22390b0953a97f285d5f732abf4af983cfc4ce4f01b899e138c720988037b"
+ },
+ {
+ "case_id": "us__scenario_048__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:35:46.903730+00:00",
+ "prompt_sha256": "99e4b79bbe41a7b315d325a46abc69ba9498be3a57979d75f54fcafed8eca589",
+ "verdict_sha256": "861d1399c9402c0eec443278d01c077f1a957b64ebc580b9831ad82513c21392"
+ },
+ {
+ "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:03:56.454492+00:00",
+ "prompt_sha256": "d51e72b2182678f4cc5d2de3f4ff0b71982f17cbd6f932f9504a2686f0989838",
+ "verdict_sha256": "ea9eca35852e5dfc057fd18f4a48acda06731c4450a8d377cfb616a04366d351"
+ },
+ {
+ "case_id": "us__scenario_049__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:39:13.929924+00:00",
+ "prompt_sha256": "f65338f628ea9b1f28e075fadc7f7ee7f8e3eed449268871cc1e11d4bfda9f63",
+ "verdict_sha256": "449d92ac7eaa937fe90b1a72d0b9eda9ee30dde9ad2c6f79c9ff840886d14bf5"
+ },
+ {
+ "case_id": "us__scenario_051__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:41:40.678682+00:00",
+ "prompt_sha256": "339e57fde1bd4e33815c49964ea8e9128eab60eeb11fe52ba340e1ca0e92b8c0",
+ "verdict_sha256": "954245423973ab966a6925c305848d6d9e36059abca4d5b295d5ff4a724187e6"
+ },
+ {
+ "case_id": "us__scenario_051__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:11:08.050372+00:00",
+ "prompt_sha256": "5ddf5d321a7d37b1950d3ce2a53e509d4015f0f31e5282c94b15f1ed0f714f37",
+ "verdict_sha256": "524f744c91fb09b511e995137a30465794edfd6c09e08517478057c5cff5ec50"
+ },
+ {
+ "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:44:18.866127+00:00",
+ "prompt_sha256": "bda9d830697a13251decb90893854a0b1e4a0e5e3c9528784b6ee5dd43d21d79",
+ "verdict_sha256": "160f92579bdaa75197452ab92dc21c2e2c997933c55dc6ba6ac1f180f84d6da3"
+ },
+ {
+ "case_id": "us__scenario_052__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:45:22.368404+00:00",
+ "prompt_sha256": "6f9f6dfc1bf65ab3059a55e3c021bc97d612452d5ca41ba98f5edbc009fffca6",
+ "verdict_sha256": "ea24754abe4dc3a73d69bef03ead75d9a392e52b0b2dd1514ecdd8cbaf7b8b81"
+ },
+ {
+ "case_id": "us__scenario_053__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:45:40.577547+00:00",
+ "prompt_sha256": "0aece15329f0f8128ef89056d347e6ee8cd8bf623de96b2784cac0ad9968f356",
+ "verdict_sha256": "3ec04ab9775d5495a4e0767dd4a05a6361c04cca7bf8aff2a5a31bc9ba5cde74"
+ },
+ {
+ "case_id": "us__scenario_053__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:47:41.220740+00:00",
+ "prompt_sha256": "c2360ffc115d1055ce64ba77c8fe3ea64bb2c6de29119f14ce957ae91377aafc",
+ "verdict_sha256": "eb6b52636f2920eadab585cbb2b329dc0b8df836b94132d0dcd980ffb28b6f50"
+ },
+ {
+ "case_id": "us__scenario_053__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:46:29.856121+00:00",
+ "prompt_sha256": "c3cc8ddb05d966f9c7889b87b924a9f9cdde79851f5e1914ccc8094db4b3dc7b",
+ "verdict_sha256": "d192a2e31d79be9970d9a776680b3836de779248dc04ab410cf66cd6083d99ec"
+ },
+ {
+ "case_id": "us__scenario_054__child2_head_start_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:47:58.781187+00:00",
+ "prompt_sha256": "ac06a81aeba13974672e68ca435af710fc1c0bf6b03c7ee20bc66c926fa4a3aa",
+ "verdict_sha256": "98972d0394e90d91587be10c820ec84c586539ea91a9cc646704c3be31d65e1c"
+ },
+ {
+ "case_id": "us__scenario_054__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:49:02.936899+00:00",
+ "prompt_sha256": "c0953192213a189f537db5dd765b70034230d7f43e1d31fc120e73eb52a748fd",
+ "verdict_sha256": "48f6300e0533f8c0a1109f8b98e4a73cc78053cb8975e8bada469716e3f94f75"
+ },
+ {
+ "case_id": "us__scenario_054__head_wic_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:49:30.330718+00:00",
+ "prompt_sha256": "ab128dc4837e05ab6282f10cc2b58b3c4142b6731716fdef879fafbbb264ddbb",
+ "verdict_sha256": "581369a3a8cf717d35f01c271231185c5f72879d6da34d600c5ce575a1e95194"
+ },
+ {
+ "case_id": "us__scenario_054__self_employment_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:49:26.652067+00:00",
+ "prompt_sha256": "2ebfe07a5c6540ac0761fbd0ae254e528f4ba984275bc7840d7cf5f9466f8845",
+ "verdict_sha256": "57247bf738aa515db9adcd330f0be7c530a7b7ece60254beb76c1be21bb89b70"
+ },
+ {
+ "case_id": "us__scenario_054__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:50:48.692779+00:00",
+ "prompt_sha256": "2ef3e32448ba649acc9e4748d10181333812ee026b7e9bf8eaa8c5c7953430bf",
+ "verdict_sha256": "98155640e04e269c42ce6ea92c4bdeaae3430e5ff8db94ef88caff0edeb42ab5"
+ },
+ {
+ "case_id": "us__scenario_055__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-08T23:51:48.469036+00:00",
+ "prompt_sha256": "2e1e73cc4f77b3727cc5d0dd99f701195195c784335a81bff72f043740e311e6",
+ "verdict_sha256": "22e9f5771d046cd0fecacc3de084c6f1a27b710035cc118a55dbc7022218cd1e"
+ },
+ {
+ "case_id": "us__scenario_056__snap",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:35:17.301406+00:00",
+ "prompt_sha256": "0841f62a55550474e31cc34cefb104a78240b982833036705488962539dbcaeb",
+ "verdict_sha256": "ebff25226d847b208498e7808663d70f8fb2540aee3dee16158b53d6810e5dc2"
+ },
+ {
+ "case_id": "us__scenario_056__state_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:16:41.534980+00:00",
+ "prompt_sha256": "bb2a5468d91dd9af3d64aab5096f59e216e9d23c8919ff94516d1ebce046f544",
+ "verdict_sha256": "7f1d11b681e01758782d470ab3c39ce75ea753d0e1d4c31f393c61214f273426"
+ },
+ {
+ "case_id": "us__scenario_057__federal_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:17:43.001618+00:00",
+ "prompt_sha256": "290a9cdb70ea9936a6a26ebdd4a25fcff76bc74b144f23b4a259f101436192b1",
+ "verdict_sha256": "1e4fc20e30413dcf9fe24966da417aac8e9ae0bc073264e5c2db8faa7f333ce8"
+ },
+ {
+ "case_id": "us__scenario_057__head_medicaid_eligible",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:17:59.769342+00:00",
+ "prompt_sha256": "b24704c65dbbbd869b75808f3d75e9d21828139588965b21f6714959545e7726",
+ "verdict_sha256": "69f7a36659fea087602df9eebcfda9b865b978b138e30f94f8eb55302fead365"
+ },
+ {
+ "case_id": "us__scenario_057__snap",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:19:45.856457+00:00",
+ "prompt_sha256": "2fe4ce665dfcae55ac68b713f956ff8693154b30ac4be90e20f06a273c274738",
+ "verdict_sha256": "fd1ddc4fdf6d6db672262811721c2125fbbb4c0e42868b2ef849ab25b9c4bfe7"
+ },
+ {
+ "case_id": "us__scenario_057__ssi",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:04:41.082723+00:00",
+ "prompt_sha256": "df01dd5a4b49a6faf2084f60412099caf41e8f4a9c0b806a1f72728538036847",
+ "verdict_sha256": "608e3fb3dd7fe97c903bf813803ecb4feaf0a1660ef7b64bac2c3679d4915355"
+ },
+ {
+ "case_id": "us__scenario_057__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:05:31.217166+00:00",
+ "prompt_sha256": "18c1faaef19c5611435805d4940339f87a17db36fc06661dc7a473487ac4dea6",
+ "verdict_sha256": "9402e362c02a61a1ebab399d78a4d54a1f0045f15e05191adc63a1d1cc58e056"
+ },
+ {
+ "case_id": "us__scenario_059__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:05:21.741808+00:00",
+ "prompt_sha256": "7630823ad57b1254214e8f0ef70da1e8190b0788c40500fbd78f9cce3df57e22",
+ "verdict_sha256": "bee1ada5651b68a206fee7f72a538e960513ca33b6fac5b36943fdf8506c6472"
+ },
+ {
+ "case_id": "us__scenario_064__dependent1_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:06:29.444538+00:00",
+ "prompt_sha256": "c085aaf95619705f66e3c49bb040b2524463cb7d4a6513d1b1328ecc183a905c",
+ "verdict_sha256": "72a5dbb23c331bdf620a5b42c79dd1bb66dea050961c9e5655fb3d68aa3c6dfe"
+ },
+ {
+ "case_id": "us__scenario_064__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:39:02.951106+00:00",
+ "prompt_sha256": "f6c40913c583a53a403540c8ef1a88bf37caac15612749479af8928364f38150",
+ "verdict_sha256": "0fb38f969bee9b4bf2758fc10bc89a32b680793d3ada65e895537222cd1887d8"
+ },
+ {
+ "case_id": "us__scenario_064__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:09:46.100538+00:00",
+ "prompt_sha256": "793f11fb2e4264c06c8bfc04b6481d95a8ca78c704dad17ad8cf3f064d713ec3",
+ "verdict_sha256": "95cc2f85c5b4783ede28e16a0339e1aa2814b71d6d5481d58540b59cb82c39bd"
+ },
+ {
+ "case_id": "us__scenario_064__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:42:31.489604+00:00",
+ "prompt_sha256": "543d9f6397bbcf2d5967b0061382e3d0484d007ad9806ce15f9f7a198b34db93",
+ "verdict_sha256": "512cb0c25fe525da266ef7444c5db3e55effe5e499a861eb99c620f97620e0bb"
+ },
+ {
+ "case_id": "us__scenario_066__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:14:23.428223+00:00",
+ "prompt_sha256": "3488c464832e9b71f70f3b888b7d5674bb772929bbeef96b0784858f5e6e3ee7",
+ "verdict_sha256": "dab99af16eee28a7b865ef93c51301b5b1576455d5e8d6530131cce5a83f82f8"
+ },
+ {
+ "case_id": "us__scenario_066__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:14:27.574924+00:00",
+ "prompt_sha256": "6dab1279c40d43495309d9c28c93ec8f44cf7229792ce2d3b612c62453ead0a1",
+ "verdict_sha256": "9e1e171ff356566c0adc18804de7165dc84df48ca50bd25db558ba5962cc4e73"
+ },
+ {
+ "case_id": "us__scenario_066__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:15:21.915928+00:00",
+ "prompt_sha256": "ff356e52dc68566f3a0791734a15571d5787e62c5e394d123a75718c6522ff07",
+ "verdict_sha256": "2344c626778ff071da05cce666d262d80cae0621b70e0915ee6cc41de04b75cb"
+ },
+ {
+ "case_id": "us__scenario_066__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:15:08.318870+00:00",
+ "prompt_sha256": "9724860d24732c5128185dfe03387ef81d99af4ee501264ec04091df93985b21",
+ "verdict_sha256": "c4bf4bff594fbc65eb8cb19015bc9fee7b6a6e1de057c5d950bd11ef00c74f1b"
+ },
+ {
+ "case_id": "us__scenario_067__dependent1_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:15:51.657540+00:00",
+ "prompt_sha256": "7cbb2316725e09d821ad990bb3a9b05e91b8ce3be2e9c3c51edf5fb18648e325",
+ "verdict_sha256": "734bb779211053f76dedeaa438166ff900e74be67e19e163850a6b3812d8880a"
+ },
+ {
+ "case_id": "us__scenario_067__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:17:12.325615+00:00",
+ "prompt_sha256": "8c7b1db90642ddf68f21b300752931545fe72caba1a597757133f8499619aba2",
+ "verdict_sha256": "e8b41e5f42e6064c546bfc46e9abc1f0489939206bb03013d1b4bdfcf3f3c38b"
+ },
+ {
+ "case_id": "us__scenario_067__local_income_tax",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:05:07.402541+00:00",
+ "prompt_sha256": "5b470a6d46dbc5ebe963e5c358aa77d19ce104c608df400f7a85f28907457406",
+ "verdict_sha256": "14c23bf7f38ae94b31c2f01fda8774837e4b599dcd43f38e2d21025c3cf19531"
+ },
+ {
+ "case_id": "us__scenario_067__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:17:35.892022+00:00",
+ "prompt_sha256": "47bf89f8663077a8d90929ab5ebe4c70b9f27a7c0a6e4055067d781e4d5fc5ff",
+ "verdict_sha256": "1349613eb90210edd684be71c1cf5d8149f64fcf09eb4e1b7f23633fde1133a2"
+ },
+ {
+ "case_id": "us__scenario_067__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:18:44.951522+00:00",
+ "prompt_sha256": "a7ce739f90dae7c9f0d3f81592ff384e8e022d2cdd74be15b6dd86fe4befbe01",
+ "verdict_sha256": "6de95aae28165e100da33a4cd7a31044b8bc3c4263cc48666d07b3fadcab5e3e"
+ },
+ {
+ "case_id": "us__scenario_068__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:18:46.036583+00:00",
+ "prompt_sha256": "46b0d4e8322b8ec2d2d470abb7f0089633a3abb859349dbf91c75b7ef1bd53ef",
+ "verdict_sha256": "9dce4b467687663b989a4922ad1232ad280e549bfcebe168bd53de6beb369828"
+ },
+ {
+ "case_id": "us__scenario_068__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:20:02.318996+00:00",
+ "prompt_sha256": "414748235f231fb9316129e0ae2b3d14ee2fe8f18a6e011e9537d14bbef7ceb1",
+ "verdict_sha256": "d7e22cc43ffdcb09f90fe0b94f51d6892460351d2151da1c4adad04afd88f3a4"
+ },
+ {
+ "case_id": "us__scenario_070__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:20:08.376658+00:00",
+ "prompt_sha256": "c070792327ce4e149633446cdfa69a16c231401f9268befe9a6305e1af6f59fd",
+ "verdict_sha256": "5e8dbb962ffb599aa2400add459c1d9c3a1d888c9fa5de60eeea904a8bbda31d"
+ },
+ {
+ "case_id": "us__scenario_070__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:21:09.292349+00:00",
+ "prompt_sha256": "c2fa1e225bb2cda9f138ac3a7640e519070907cf5dad0c9782e6b943f63ec164",
+ "verdict_sha256": "e21b0d38d07ba0dc368bf37f2aa6ce0cf3859df1cf9b095dcc5550b2c292b259"
+ },
+ {
+ "case_id": "us__scenario_073__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:21:22.552143+00:00",
+ "prompt_sha256": "b4e5052fbb45e2022475ac12e736734229b00c784c57ebd2130decb8e9f69b9f",
+ "verdict_sha256": "56073985c932e4c369df4e016b5afb8626c1c6415f57255a055e8efee73d28f5"
+ },
+ {
+ "case_id": "us__scenario_074__head_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:22:30.876116+00:00",
+ "prompt_sha256": "f0a99534476b8780cd94916f15dda490a55fd7c013917070cf3295d8b3e01d4a",
+ "verdict_sha256": "e21095233ff1c9c56ba96883ef30dc220c12b6a9082ab80b762fefff10308762"
+ },
+ {
+ "case_id": "us__scenario_075__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:22:31.955704+00:00",
+ "prompt_sha256": "9c7bc63ca553f35e6623ed395e2b082752cabd103526192b681e44b5f0fd2a88",
+ "verdict_sha256": "62c7d4602bb2d32f6f3156c45915eeaf2da6be2f658a1772f2d0496f70e3cff0"
+ },
+ {
+ "case_id": "us__scenario_075__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:24:09.179926+00:00",
+ "prompt_sha256": "7d1c1f7e262b9e6a0c67063f196ea4d86b385e66f4164bebd70d580ea769c936",
+ "verdict_sha256": "1429898520996c496b6ddf693ce517f323fccaa59dfaac79c35275a41ed74a9e"
+ },
+ {
+ "case_id": "us__scenario_076__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:25:42.840391+00:00",
+ "prompt_sha256": "dc3b89eae9ffc45291dbad75cc6a7a538623cbec700edadf9a1c8e23a6ba919f",
+ "verdict_sha256": "e83e13c40d5eaac576fd58938670de1bc1b9bb20eee203858de6a3ab219727a6"
+ },
+ {
+ "case_id": "us__scenario_076__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T18:44:13.267676+00:00",
+ "prompt_sha256": "d37ad790c66a0ba9491f8f1e8a5bc247b854d6077cea89abf6ead32bdff41f22",
+ "verdict_sha256": "8e4680ae3a223014470cb19b74e864522c20ae9e4a3217b126068bb5a868960e"
+ },
+ {
+ "case_id": "us__scenario_076__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:26:27.137036+00:00",
+ "prompt_sha256": "bcaf490a0941ac20f02f335d9b5d4f80176826dbf5cd68726a0b84ddbf296c3c",
+ "verdict_sha256": "b6f89c06b7b067b71609f65badb0f5362a037d32219c89ce0f18cef73fa79534"
+ },
+ {
+ "case_id": "us__scenario_077__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:28:23.272263+00:00",
+ "prompt_sha256": "cbc6469c3f879b5059834f1fdce059167be41e86e69ba58a79345011238e7c65",
+ "verdict_sha256": "19bcb97950e18f6ce00439f3fb7f9ca0530fc7fea0ad55e506c9edd9580f4fa2"
+ },
+ {
+ "case_id": "us__scenario_077__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:21:03.164972+00:00",
+ "prompt_sha256": "dc10c82a8bd5cc2bba4ac13701fd6cf52aa8f4ccbed4f72d7ea6b67e71c5efbf",
+ "verdict_sha256": "ac25f9a957e6df1a804ffbc0826fb5f27581670aedfe4512f1b136e2654f5e6f"
+ },
+ {
+ "case_id": "us__scenario_078__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:29:00.560849+00:00",
+ "prompt_sha256": "d0462157251cae830add57246b1b10cd7ade4bfce2398779cf587a7aafc3a79e",
+ "verdict_sha256": "b2d91b793cbe19150b176bbeb6204a4b7809879f36bdfc1042a02906f84bff97"
+ },
+ {
+ "case_id": "us__scenario_078__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:29:57.275560+00:00",
+ "prompt_sha256": "bbff94fca330f1b1a09f6078971c8e7827ecc23960ea0bc7a876e8a9d165f7d5",
+ "verdict_sha256": "2c3b31657284cf46b52990d81c1ee7cf26f4c535ad1f714e80210a7952bb23c5"
+ },
+ {
+ "case_id": "us__scenario_078__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:30:39.613657+00:00",
+ "prompt_sha256": "bba3aa10ce12471805a4c87dda26a41b9bf337a5565ca2ff88c4e660d7519cb3",
+ "verdict_sha256": "2fb4cbdb4f167b71c15c97d78f0408a783aa69dff95c8f8dd78b170b850d6dcb"
+ },
+ {
+ "case_id": "us__scenario_079__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:31:58.349534+00:00",
+ "prompt_sha256": "358b7d066755e939451e115bd684051e1dee80621ce4855a44e8070dde7c2246",
+ "verdict_sha256": "5d720888ea025cbb7a906771e21385bd1dae1bc787b4fc1ef7ea92b2a336ad43"
+ },
+ {
+ "case_id": "us__scenario_079__ssi",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:31:36.737431+00:00",
+ "prompt_sha256": "933e09f2a7ad4721d29414b4472d27d7e77e7d0f3c80936477f5d2c93156d043",
+ "verdict_sha256": "13431be4f0c9bc6fef0ce3f18db21bcc0fda87aaf5e500f44f26573fab6930e3"
+ },
+ {
+ "case_id": "us__scenario_079__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:32:50.755375+00:00",
+ "prompt_sha256": "af85643d6d3badc459565b10d78abd1bf9dc3c31b61196c9dcf6710924995945",
+ "verdict_sha256": "ccdc6b959d15ad34e6781e8d45b623ec61eaee88f9194bf000b6e4be3f6f9739"
+ },
+ {
+ "case_id": "us__scenario_080__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:32:18.853137+00:00",
+ "prompt_sha256": "8db9ea4765f350366b1609d0ec8a322bc46e106eaa2ea8df277872bdf357bb09",
+ "verdict_sha256": "89e224563d973dcea92e8a030b2ae805db18454dc8d918115005a5e3af0449b6"
+ },
+ {
+ "case_id": "us__scenario_080__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:34:15.185699+00:00",
+ "prompt_sha256": "6f1f87267f9c7160cd0cf2e19417e42be61db743242b5f3ac6b4457378fc09db",
+ "verdict_sha256": "00461d72bec0c63e44f74e1119a5a051bd8db0d509dddc95aded9136c2c92cf1"
+ },
+ {
+ "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:34:39.179419+00:00",
+ "prompt_sha256": "01f58ab354674e5edeacc02fda7ab4c7bd8992218e5223001c288b8a1f24791d",
+ "verdict_sha256": "11819fede6b291f15a7feb97469c3975b1ebd32c53cf3d11a697df64deae28c7"
+ },
+ {
+ "case_id": "us__scenario_081__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:36:05.565616+00:00",
+ "prompt_sha256": "46c6f5fadadab8ee0a97416418cbce9def84fcfb84513efda1a389a2b9062492",
+ "verdict_sha256": "ef2fc03439ff51bc446cee92fabe93e28ebd5e988d31d2173341a7ec0fb8c1d4"
+ },
+ {
+ "case_id": "us__scenario_081__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:37:14.226310+00:00",
+ "prompt_sha256": "6a2d1758f67f045d081aee8b67c8f223fc6b95b94fa10ad6fa3b48cb308d6857",
+ "verdict_sha256": "434ef3ffad3c4c9e92e1d1a282fd3a08018f1d7f5cf2987227bb60b86a733995"
+ },
+ {
+ "case_id": "us__scenario_082__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:07:44.449916+00:00",
+ "prompt_sha256": "e2cdaab17ab3730ec11c42b32e066e2f21c5b0a9bb0351cf7eb7bde3e3bc2fc9",
+ "verdict_sha256": "2b3aa05db39d4f909d50328dbccc8de00a5d42e4ffab09fb2ef9f31b2608ce72"
+ },
+ {
+ "case_id": "us__scenario_082__local_income_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:37:29.542160+00:00",
+ "prompt_sha256": "7e5e56f143679d05ce5e6a85312929eb7775619d74092b959eea6c0c5ce60562",
+ "verdict_sha256": "12dbd03b078e5536dfd64d9a2361d3da5bd6103062e9fb62b9464eee5a48ae0b"
+ },
+ {
+ "case_id": "us__scenario_082__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:41:27.145196+00:00",
+ "prompt_sha256": "748242baa478d1d1c9209b9c4e91d29089b5f17b93f7fb49a25ca783d8bb5100",
+ "verdict_sha256": "75e0dc06d14f919cdee0a2e82a8611ea2babb73c2d62d4dbe1d507ccf9e5edf7"
+ },
+ {
+ "case_id": "us__scenario_082__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:11:22.623745+00:00",
+ "prompt_sha256": "b35e2afeac00102faf9d716554e4c800a8ee4dc2e3b7d0c6feb2d705b51ad7b2",
+ "verdict_sha256": "248bde3027955427bc3b54651f88b86d447a2f748731ff0a17ca1e3a70479bca"
+ },
+ {
+ "case_id": "us__scenario_082__state_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:12:39.904704+00:00",
+ "prompt_sha256": "106165a155117af10eacca2a00b893d8f4e7a165afc92e7445560f2aa81b0bac",
+ "verdict_sha256": "48ae58d8e85ab2ea0ebabdfa1bab08f5d8e7cf4ffb2a8fb0dbfc8b42bed2e14d"
+ },
+ {
+ "case_id": "us__scenario_083__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:42:56.119980+00:00",
+ "prompt_sha256": "c4391157a1e537f41f75ed7fd2cda7074fe1572909fcd9100b287dbc18f0eca2",
+ "verdict_sha256": "a1ca26d419510251a0ce99bfd8af3bd41c00f754a7528742228d27c0b4bb2eaf"
+ },
+ {
+ "case_id": "us__scenario_084__head_medicare_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:43:14.074826+00:00",
+ "prompt_sha256": "8197979f49d7305ad0ff467b6d11d27ccd828c3d2d098bf90d0e376a4eae9e83",
+ "verdict_sha256": "8a3f550550750e55f2faf3c701a271d21512800edff4229255fd002cb710362c"
+ },
+ {
+ "case_id": "us__scenario_085__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:13:47.471732+00:00",
+ "prompt_sha256": "a273818e33f8e7f7f84955fe2cc32f5326d7c8967ae0818d79f5486b96eaa222",
+ "verdict_sha256": "ec50a0771a8e881f257d2eefcf709894a42ce36c7804ab120237b35e3dc7f7c8"
+ },
+ {
+ "case_id": "us__scenario_086__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:48:12.100662+00:00",
+ "prompt_sha256": "9272ab85afceb414fc2f8c418874532f6d91b36a6570bb5231ed3e3dd7f3b993",
+ "verdict_sha256": "225f0748e81e9e4434bea454fdb99a5b496a67fd9504f477508623759ea52cac"
+ },
+ {
+ "case_id": "us__scenario_086__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:50:17.854306+00:00",
+ "prompt_sha256": "5f8f2ae329eaae2811d349cb832db188a7c721ce9a507f2b9d213194b9b9d82b",
+ "verdict_sha256": "bb0c19cbd2b1e7d51dc3eba71e3958d7cfa527bd3bb652811738d50c1c522ed5"
+ },
+ {
+ "case_id": "us__scenario_089__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:49:56.726774+00:00",
+ "prompt_sha256": "97bc7070d9808eebcab4580f0b8d2320fce32585479156eb233be13a590127c1",
+ "verdict_sha256": "74144742eaf21fb93146958ad63f28eac70c7334dfef1fcb41a826d8d349c794"
+ },
+ {
+ "case_id": "us__scenario_089__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:51:03.659195+00:00",
+ "prompt_sha256": "3f665497df67834b1bf33fe1ad8ba3bd0b8d2d2ced08e014dcc7d4296efc4359",
+ "verdict_sha256": "f891b02dc2c95d4981e136811d6076c9c4a0a919ae3288364dc32a39635c9679"
+ },
+ {
+ "case_id": "us__scenario_089__self_employment_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:51:00.702994+00:00",
+ "prompt_sha256": "31b4068a5c1b9409fc907ce06fdba962d424a2b20fb8f3c88bb21d7053b5cc3b",
+ "verdict_sha256": "db7bf9ff52dde77a27ef2490a502d241c983fb3f8662535af91719a181ccaa38"
+ },
+ {
+ "case_id": "us__scenario_089__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:52:35.229102+00:00",
+ "prompt_sha256": "966d4e9be835310eac25f9dc755cbe44e9329f5e6be6d2adcdf3ac8609ddfcfb",
+ "verdict_sha256": "819d83f792344de3bf9d870cd7d738d9129575919ba75ac679bd2a870f8e81c7"
+ },
+ {
+ "case_id": "us__scenario_091__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:52:51.068406+00:00",
+ "prompt_sha256": "25abe3dab1c432cf24370f63e280cd751621416abd44eeb48568e7a375503556",
+ "verdict_sha256": "65b18a70ecd45728ca9402a7b6cc7e4b4668f3b3c8f5fcbd04c038ec4fb97d0e"
+ },
+ {
+ "case_id": "us__scenario_091__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:54:52.785766+00:00",
+ "prompt_sha256": "ac9cd8333e6a2cbddf3cfb389ca1f9265fe2c238ad311954200e8db16abbf90e",
+ "verdict_sha256": "b6e8b55b903170f86e341c8a3aee723b1d174d78659fb811d979c404dfd65854"
+ },
+ {
+ "case_id": "us__scenario_093__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:24:55.980556+00:00",
+ "prompt_sha256": "2e090f4f4dcda49240d4547494268853936b62f8dde3dfdc2ccf4238633ffae5",
+ "verdict_sha256": "64c3a1b7c1e9dba12bd5460cf18033a610f4f11ed3c685a1977986ef3342c086"
+ },
+ {
+ "case_id": "us__scenario_093__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:53:41.084029+00:00",
+ "prompt_sha256": "38652b10eb814185590450f18abe2a4f99e8f38cfd8834c0fac5f92ea6c4261c",
+ "verdict_sha256": "90d5369aeb98e04288cb4edc99f3d86afc3432000627f5ee2fc3f95138dd95c0"
+ },
+ {
+ "case_id": "us__scenario_093__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:27:21.537119+00:00",
+ "prompt_sha256": "1abe54eb43624689045698b8386a96cfde175c1d2e3a95d3e886a1fded6e1621",
+ "verdict_sha256": "3bb92b597392168d3797344d6d533f5469324efe8f9bbcc246eed79e132a000e"
+ },
+ {
+ "case_id": "us__scenario_095__self_employment_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:55:18.294162+00:00",
+ "prompt_sha256": "3c8a6a24a49d2fc4d33018793b727659658fe8a35cfd1a517ce44fdf107b8d4b",
+ "verdict_sha256": "38f626745c3acd474cf9d1892074757a174dcdef67640ee5d9f66b152842ffda"
+ },
+ {
+ "case_id": "us__scenario_099__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:15:31.747428+00:00",
+ "prompt_sha256": "fb80ff34599d7414e348de256c9f403218b26edaba12015be17d764626f203e7",
+ "verdict_sha256": "18524df596840bae585909732f4096e48a30a6d36f506916cf59cb8a3bf3da89"
+ },
+ {
+ "case_id": "us__scenario_099__free_school_meals_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:59:02.138750+00:00",
+ "prompt_sha256": "d7d1fdf83cad78a8e90e7d354f0f22b2bd3c1866d329379ab5d1ad8e079a199a",
+ "verdict_sha256": "84af914514a8e4bdeea05bbaec708f44440faa2a6975997dd5d3c4c1d7ea51f3"
+ },
+ {
+ "case_id": "us__scenario_099__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:59:16.595063+00:00",
+ "prompt_sha256": "fb1328b0a9b09f1a008d02534a2df6ab58e48d1734efb18dcdf6e138e45726db",
+ "verdict_sha256": "2256c2ad86cee224488587f5d01b2f5664a5cd550e683e180d2ed3e5f8e8b01d"
+ },
+ {
+ "case_id": "us__scenario_099__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:46:49.657279+00:00",
+ "prompt_sha256": "4e70fc18edd1d27e2ef589ae02325425bf0a8115221edbad4fa94bcaa38dc571",
+ "verdict_sha256": "5a96439af716b7d43516665a1bdcc7db7d9b3f72cda484764c67f88daa996314"
+ },
+ {
+ "case_id": "us__scenario_100__child1_head_start_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T00:59:39.446880+00:00",
+ "prompt_sha256": "49c5267e1955e218824edb896ef6afe8513c15900fcb3fa8162d8464e1d1dbec",
+ "verdict_sha256": "a1bc7cfcb12a742bb47862d3d6a680916ba4dd02badedb72075603aea79cbbd6"
+ },
+ {
+ "case_id": "us__scenario_100__child2_wic_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:03:57.450213+00:00",
+ "prompt_sha256": "34755e3b9718b10316b26a50b9417140686100682ea3f752cb3c65eb234f96e0",
+ "verdict_sha256": "163409e6e0817d44d1e1be2755c836b6154e25357a96be94c065a16cfd557b96"
+ },
+ {
+ "case_id": "us__scenario_100__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:05:54.209198+00:00",
+ "prompt_sha256": "dedc78aa47bf294cc1f15018fe354974a0d22a67d0997d1882ddb3cf3c1868dd",
+ "verdict_sha256": "a9babaf7cdac3da5f128f36705badf7b3efbaff1f8e94ee9e88886406c5421fa"
+ },
+ {
+ "case_id": "us__scenario_100__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:06:08.121531+00:00",
+ "prompt_sha256": "48e03b27b24ed322a410324bbd575170ecdc529b7ac7a62452345654b61918c5",
+ "verdict_sha256": "b517187637972fb7d544a8f1f29773907108f9f95dd54d08474954e9d2bb9b02"
+ },
+ {
+ "case_id": "us__scenario_100__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:08:24.120656+00:00",
+ "prompt_sha256": "5546ec301eba6b35264477f4f5cbe928a1fa521b5ee175a8e066e8b354731893",
+ "verdict_sha256": "74f15fab7d138af0040de665895e5ef17bdd031bad67b387495112ceffbfdf55"
+ },
+ {
+ "case_id": "us__scenario_100__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:09:55.581168+00:00",
+ "prompt_sha256": "73a92e3143934b51d7fc21569bae2d19e4cd1ff6cd3b9fde63771b666ad05de4",
+ "verdict_sha256": "19c2d6d31341942983ad670f4247af19ea823faaaadd7993cc622516b59abe2c"
+ },
+ {
+ "case_id": "us__scenario_100__tanf",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:09:47.565388+00:00",
+ "prompt_sha256": "c70fb95cf994c9ce31e0bdf5b07096f9b3f04666580b32c6ee2ad6eac2718036",
+ "verdict_sha256": "61106f7a525f1e786079128133e9b650038d4ed0b14f2371ec1a7f69cc35fd4a"
+ },
+ {
+ "case_id": "us__scenario_101__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:11:42.813357+00:00",
+ "prompt_sha256": "ed69208c41533212218261eadd9127effbc19334becce883d0b24af3cdb87c09",
+ "verdict_sha256": "6ded4f3cf867cc12816d956c9344febe03ca1b4aed9b2e0ea24918d6d26ad3e1"
+ },
+ {
+ "case_id": "us__scenario_101__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:10:13.137167+00:00",
+ "prompt_sha256": "bed608f5de9caef1a933f0990d62a08367b3615af714ebf9aa0aa75b6e3f1fec",
+ "verdict_sha256": "99fdc0ed8e493e4a348c42dc035013c006a9c0d4463c25e46c7c3382b1d97e7d"
+ },
+ {
+ "case_id": "us__scenario_102__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:12:49.709943+00:00",
+ "prompt_sha256": "1f4857f021de2c2cc2a579256a00a01f633850229b58de0d9df7092c5fa2b928",
+ "verdict_sha256": "8c9715481ea35eab3a5881a01091de3540da5a1ea5f41de266a0fe8024b263e4"
+ },
+ {
+ "case_id": "us__scenario_102__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:12:22.374387+00:00",
+ "prompt_sha256": "9e82e5aad615feda16e19f80961d7cfa34d536d6115c6f5684366e8b8a984de5",
+ "verdict_sha256": "49094627773fcf98ec5b1c460d43f8bddc2f72e90d5dec41ec2daa4a583f3605"
+ },
+ {
+ "case_id": "us__scenario_102__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:13:41.822143+00:00",
+ "prompt_sha256": "274d7a14d83b6264b2c437f0754c09843e9d2f6c834a67d5331077e1935eea38",
+ "verdict_sha256": "67b37ee4f21d701c7986e8bf3f51ba84db82ec6f1d782b2da8d22cc989b5439c"
+ },
+ {
+ "case_id": "us__scenario_108__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:13:35.200215+00:00",
+ "prompt_sha256": "db5a96055dfef40d51d99ae837e5b5992280904d4fa813876ea34b4533686a91",
+ "verdict_sha256": "ac8f0f7044564eada7aa064f3a30c5e854fb06c21936ca03bd2b096986840fda"
+ },
+ {
+ "case_id": "us__scenario_108__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:14:08.678185+00:00",
+ "prompt_sha256": "31501e3f7b6106c30f3e88cd461b81b1c25c0f2778a111b77734291864028f81",
+ "verdict_sha256": "07b5f9a7fd9978bfd4172f80262fee544dbb2b871f1cb1503bcef419707fd653"
+ },
+ {
+ "case_id": "us__scenario_108__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:14:57.973301+00:00",
+ "prompt_sha256": "b3f71e02d3c2ce99092eb763138308233e74ac28a9705b7fafe2f1b1534c0e38",
+ "verdict_sha256": "ff1af8ca8475b1bed5d17a3af7e079f2a0c18fe1424172b5586c47b920baae9e"
+ },
+ {
+ "case_id": "us__scenario_108__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:16:19.880626+00:00",
+ "prompt_sha256": "b9519c04fdc693206d8d5778de6cc014edfb68d02afdd645679fb9ab8bce80e0",
+ "verdict_sha256": "52e48492b54957b7ff7b983bc959cb1472a5cdb46a5f115baa78000641a9f88f"
+ },
+ {
+ "case_id": "us__scenario_109__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:16:21.610844+00:00",
+ "prompt_sha256": "acbdaf149b8a09bf55f5c20dfa7b12277165727c3023b7e8f620335d331c9e18",
+ "verdict_sha256": "02d325ae1ee91abd23a80ea484eced2a6f169ae98a10eaa328c6de768434fe4d"
+ },
+ {
+ "case_id": "us__scenario_109__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:17:59.525872+00:00",
+ "prompt_sha256": "3e5c7fb6d09f3c8a811a6c153ae57b5245e907bfe126c17ee48929a688320bc7",
+ "verdict_sha256": "9b7a7dbd91b86c0989d9ca17060ed9858a1f75c6f62857245ff6c4cae51e4d0a"
+ },
+ {
+ "case_id": "us__scenario_110__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:22:10.567304+00:00",
+ "prompt_sha256": "68fd6bb9d44c315a4d869603a9ffd143e99b64430d2a4bb66db58868ddec819f",
+ "verdict_sha256": "b8e7ffd62f3b7a0ce5f9ecbcb59adbd9ac09f97502b40b47b86e41b094947765"
+ },
+ {
+ "case_id": "us__scenario_110__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:21:07.166514+00:00",
+ "prompt_sha256": "abb5e050dded944da178dc2c39c85ad041fceccd4fa99f5374a9ce6d010f4f53",
+ "verdict_sha256": "991c2b3ed25a7714261e4df135f08e750fe84fb1af72cc83aa2849cb711f8969"
+ },
+ {
+ "case_id": "us__scenario_110__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:24:07.145488+00:00",
+ "prompt_sha256": "43aad489d0c1a53b8a030b85f0aa0485f9eb77cdf019ccef3e3e4843f135cd7c",
+ "verdict_sha256": "c0f68bed3abfb94d9eb9887ec8292a0b8a0cbeb761ea9841c8cab96a106ec9df"
+ },
+ {
+ "case_id": "us__scenario_111__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:24:52.730945+00:00",
+ "prompt_sha256": "a430153ce573528490e51ba0f40315658e0569a761c172d4fb55e336e007df14",
+ "verdict_sha256": "f1bcbd2a52d30f24f2da19e19df059578e4b017e943372f763c34343a48a5bad"
+ },
+ {
+ "case_id": "us__scenario_111__spouse_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:25:03.364007+00:00",
+ "prompt_sha256": "10eadd161726da6ef4678285641680ba6165cfb04ff1f3e87be611520f7ed144",
+ "verdict_sha256": "b99d3dcc48b45ae2fdd187409fc6601176da2db82907c6abd08a8228e4e5cf65"
+ },
+ {
+ "case_id": "us__scenario_112__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:25:53.519621+00:00",
+ "prompt_sha256": "95a3b68ff7aaee81a894f49e8cfbf49f9874b2578840b33e11a3d0cd65f72814",
+ "verdict_sha256": "f9e281d4620ae229345fd73a1a1d3cb88c1f5ab516fe82141043676f97d74bfd"
+ },
+ {
+ "case_id": "us__scenario_112__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:27:47.216798+00:00",
+ "prompt_sha256": "379654f6324baa1470449fa2a120c0790bed2a3d81a702fcc7ae5977673a12fd",
+ "verdict_sha256": "ac170d3f4cbc670f941a5c58f3cda35f1e513a9e2cd4ac13666f93547a79bb25"
+ },
+ {
+ "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:29:10.069802+00:00",
+ "prompt_sha256": "d8becc4ab482ebf61fd3b9f3225734b03480be064e2f6ac852f867713bc6363b",
+ "verdict_sha256": "1330438b2d1f63b43b908ba931b410f08e1613df5e3ad00c7e8f9202f7d6975f"
+ },
+ {
+ "case_id": "us__scenario_114__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:29:39.683360+00:00",
+ "prompt_sha256": "704f4a71f89ae1cc31c22e019aabcc6733ac38edd6ffa8d92d8f3527fa8b86c9",
+ "verdict_sha256": "1324cc2803927a67fa537f5370bcf3873c2261eb9318842bddee4f47469cc0ec"
+ },
+ {
+ "case_id": "us__scenario_115__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:30:44.516172+00:00",
+ "prompt_sha256": "10a710484b385b25501f9c36b0efa3a056388a2f2a2f4a4fe871121f10b2f7e3",
+ "verdict_sha256": "aeb8191318a24758a799134d73edfffc608d24b7b974a76f3af488e96cc01a24"
+ },
+ {
+ "case_id": "us__scenario_116__spouse_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:29:55.830032+00:00",
+ "prompt_sha256": "c61163baf718458e30f529ab3f9d84f8c6fcd32dd25dd0debc327713d83b9da3",
+ "verdict_sha256": "c71899c630c2f0e9f6cea6d20f3261faf36b427cf5d70c0d32395f1b4e481f19"
+ },
+ {
+ "case_id": "us__scenario_117__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:33:34.425206+00:00",
+ "prompt_sha256": "09c0552d4e79e0ebf6414adfdbbdff8386bc4ccde56909916a15cf797970ae4f",
+ "verdict_sha256": "3162677f98080f94f6cb0728beb1dec62f5d1db1c2e2802abff91cf51414a2e2"
+ },
+ {
+ "case_id": "us__scenario_117__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:31:30.689065+00:00",
+ "prompt_sha256": "c335e18898b96ffe23ac22315159e03f576dde59837f0d53c1823e526b1c7c16",
+ "verdict_sha256": "0e50359baa35a7eeca90dda0d4e5001dc4174f82e94c4a7e9b95419c680860eb"
+ },
+ {
+ "case_id": "us__scenario_117__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:36:31.065002+00:00",
+ "prompt_sha256": "db635693d7bc860de41c06cc4ca468a09d16932913c718264891e5b2b8b9042f",
+ "verdict_sha256": "798a891a9d7930043fa8daa6edacd7a9b0a55d6070a6d72dfd4e75386f3df08c"
+ },
+ {
+ "case_id": "us__scenario_118__head_medicaid_eligible",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:33:45.022788+00:00",
+ "prompt_sha256": "e4ab3e94a6988cdc765f69cc7521cda5d6deeb91acd4732a887561f9863e2e87",
+ "verdict_sha256": "4b659d697f6a85093b208c45914fd9e67cc0c04e9c9352bdab973ff2d31e5a8c"
+ },
+ {
+ "case_id": "us__scenario_118__snap",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:39:10.605620+00:00",
+ "prompt_sha256": "f90f975a772bca6957782b86a5f23dc2496c75cc37ca230676f26306de805014",
+ "verdict_sha256": "9e3071993def3b401623d836b3c5b9f372ab908d7dc02a1c5a5d32db88d5d858"
+ },
+ {
+ "case_id": "us__scenario_118__ssi",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:37:35.825184+00:00",
+ "prompt_sha256": "294d6c5c9dccc91c2f8137be0f4a8069dedbaee62f2ebb4966070df98d3305d3",
+ "verdict_sha256": "229093fb0cf25b2ecb4477f957041bd7aa80cdc7140bcb1f2086ece5a9ef0a30"
+ },
+ {
+ "case_id": "us__scenario_118__state_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:41:10.126431+00:00",
+ "prompt_sha256": "73de087579ec693e6b08083dc403790ee946b2b2e25c15a1914b88ecca39c9f9",
+ "verdict_sha256": "3f4e0f7e3d6625e5e4632e0e16d77393e6e264c58b8aa5511a853fc5ec2055f4"
+ },
+ {
+ "case_id": "us__scenario_119__federal_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:44:25.406626+00:00",
+ "prompt_sha256": "c86b4c5002d60ef35308098bc91028e3720d2fe256cd937d01a8321d4b78e117",
+ "verdict_sha256": "cfa26c1a66e81f4b3bcb62267481ecd63558efd4d9d79e3937e53b9096828cd5"
+ },
+ {
+ "case_id": "us__scenario_119__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:47:30.156708+00:00",
+ "prompt_sha256": "08c3b45a7808144983f148fe5477465a75d979819c6edee3037f3313e9e73c7b",
+ "verdict_sha256": "ec826f2d7459dde71206fc346cc4f3cac0c54c68e2bb00297312d56f39b2a444"
+ },
+ {
+ "case_id": "us__scenario_120__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:47:46.213458+00:00",
+ "prompt_sha256": "6d5a80ab6f4dc9596c6bf051a1e79e6e54a51e510f7bf7b9c1127e82d16a7255",
+ "verdict_sha256": "85b90ac023b424b1ded692bdfcf1264c3057abec7b76a9a9f5e969be2c1d7f1a"
+ },
+ {
+ "case_id": "us__scenario_120__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:48:55.261774+00:00",
+ "prompt_sha256": "2450d5a1ff69962c08526521840a45f0c7f84dae8c9387a57fb6fbe9021265d7",
+ "verdict_sha256": "f593478dcaf6e4ab985215ea0a73b1b1497befaa5550aff74c5e3f079eb6c91d"
+ },
+ {
+ "case_id": "us__scenario_120__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-10T06:26:05.226803+00:00",
+ "prompt_sha256": "f4210559b88077959775e708c92318c150ffa2a3122e3b52c0b62261db8a031b",
+ "verdict_sha256": "e423fe128a5dd75e9e5311634511ace9d93026dfae1b11c6279deb11017b2d1c"
+ },
+ {
+ "case_id": "us__scenario_122__federal_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:52:20.042761+00:00",
+ "prompt_sha256": "a8665f2b50ca125ae9645b91e160449da5abe95dd954c6aa347a932bd2280eb5",
+ "verdict_sha256": "1bcad0e4f0b82d90018af01ef69e8f82a88b28e4c8cc57ecba9561a54461e5fd"
+ },
+ {
+ "case_id": "us__scenario_122__state_income_tax_before_refundable_credits",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:52:34.037808+00:00",
+ "prompt_sha256": "75f588c4d9eb58d82013b0605131ef9aba877b373fb4f090efb382180ba8f9c5",
+ "verdict_sha256": "0fc5ffe168fd21ce539b89d2bd0df56c99122efa86d0b0de21666624eb2b97bd"
+ },
+ {
+ "case_id": "us__scenario_123__federal_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:30:57.145495+00:00",
+ "prompt_sha256": "82221be4970b2d7513bf2939d0aa78f1e6580349c0213fb56e05970a2515fe74",
+ "verdict_sha256": "4f6f31d6328752b8321dd1bf304c4fc915568398d887aef734e362c41ac99e02"
+ },
+ {
+ "case_id": "us__scenario_123__local_income_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:52:49.987310+00:00",
+ "prompt_sha256": "20e77243a50a63813619ff778a75cb454aa55599ed980836cd48febee90e15d0",
+ "verdict_sha256": "42e5e539f01cd09f5ec915d9614e97528dae7209472e7d05ad0a89e15e0efe04"
+ },
+ {
+ "case_id": "us__scenario_123__payroll_tax",
+ "group": "isolated: setup-token 2",
+ "isolated": true,
+ "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T01:53:22.043128+00:00",
+ "prompt_sha256": "d21e14fb430142fadc31cfd53c286f4667e1b13cea2947101e69a8fe5e81fbec",
+ "verdict_sha256": "ef1083a6758848ebdec794e2164697bd0388d395fdac65a8dc01cf22f1579f48"
+ },
+ {
+ "case_id": "us__scenario_123__state_income_tax_before_refundable_credits",
+ "group": "isolated: subfleet lane claude-18",
+ "isolated": true,
+ "judge_account_declared": "claude: (subfleet lane claude-18)",
+ "judge_effort": "xhigh",
+ "judge_model_reported": [
+ "claude-opus-5-5"
+ ],
+ "judged_at_utc": "2026-10-09T11:32:09.911176+00:00",
+ "prompt_sha256": "21d9ebdded056874eb0dfd5b9a639f75fd70fcaac85ec566f255b9a8607b781e",
+ "verdict_sha256": "ad43142d632889e7c01321c24189a6c1c30fac9148f5d24c28123cb66fa7b12e"
+ }
+ ]
+}
diff --git a/docs/haiku55/judge_verdicts_20261010.json b/docs/haiku55/judge_verdicts_20261010.json
new file mode 100644
index 00000000..83e83fe8
--- /dev/null
+++ b/docs/haiku55/judge_verdicts_20261010.json
@@ -0,0 +1,455 @@
+{
+ "note": "The verdicts release dashboard-data-20261010's adjudication entries name on the cases Claude Haiku 5.5 re-opened: for each committed decision on such a case, the case's verdict in the exported stage, read by scripts/date_haiku55_judge_verdicts.py from verdict.json and its sha256-bound verdict.meta.json. The verdict each restatement replaced is release 20261006's entry (git show 9ce4ade83829:annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json).",
+ "release": "dashboard-data-20261010",
+ "base_commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "cases": {
+ "us__scenario_005__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:18:27.193218+00:00",
+ "verdict_sha256": "bb5560621a65bbe5ac1c3db9ef0ee746ee02c57f1ffe11cccea7b0bb26675d11"
+ },
+ "us__scenario_007__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:24:05.267634+00:00",
+ "verdict_sha256": "945362b7e820ec8b6f606a63583bd85f7e7683b46e94ead1e3a4cd913e54b6d4"
+ },
+ "us__scenario_007__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:01:12.293170+00:00",
+ "verdict_sha256": "ee385050ee1a01f48f66e1d6160621e1867c15de9faac56439cf0db3634459c6"
+ },
+ "us__scenario_008__state_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "reference_later_law",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:30:24.349878+00:00",
+ "verdict_sha256": "bf20e47d5085be40b1e41050d951b428c86fe531dcb1e469a366f7e0e2dce003"
+ },
+ "us__scenario_015__local_income_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-10T05:58:15.427599+00:00",
+ "verdict_sha256": "e2f324f633297e6cfcd3832de8cba9d5a71b8f022d3bb0694dd30a0cb6e9ce91"
+ },
+ "us__scenario_020__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:40:06.525009+00:00",
+ "verdict_sha256": "154da3be56f8792356ad68c1df2e687bdce6098354aa78c42b55df707222504a"
+ },
+ "us__scenario_022__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:44:46.816224+00:00",
+ "verdict_sha256": "7beb1c74f6cf134e461424cb7adb52ff8d2c0248268316317bc8c1f2a3e9ea9f"
+ },
+ "us__scenario_023__head_medicaid_eligible": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "categorical_eligibility",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:49:07.881342+00:00",
+ "verdict_sha256": "d1263a6c3cfb4d629b05ed061dd0c7f9d1ec173c4951232fd0f7339d5b3745cf"
+ },
+ "us__scenario_023__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "age_disability",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T22:50:54.620139+00:00",
+ "verdict_sha256": "cbd9abdb5318469c30d4aa62f2cf3dce7a5dc43d13b67553c9e3e515f5aa90c1"
+ },
+ "us__scenario_030__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "categorical_eligibility",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:08:47.548119+00:00",
+ "verdict_sha256": "8eb4fa2cba889e3d8b0516646fc53a581af94b342f06ba49e6240e099bdbc18b"
+ },
+ "us__scenario_032__payroll_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T23:07:05.362213+00:00",
+ "verdict_sha256": "bf538655ee75fbab32127c3d2edcc64b8ddfc3be81c00e919afb5b2a1e8e01bb"
+ },
+ "us__scenario_033__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T23:09:02.707654+00:00",
+ "verdict_sha256": "6337684e8c8766368358f1f37b49ba71a0915cf10b9b3183b60bf5882fc8be0c"
+ },
+ "us__scenario_042__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T23:25:33.269671+00:00",
+ "verdict_sha256": "780085ada860c36c43125ac03a11dfd43557032cdf8c505d0b4f3c705e5e9676"
+ },
+ "us__scenario_042__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "reference_model_issue_fixed",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-08T23:31:55.110227+00:00",
+ "verdict_sha256": "68eed26f001c269c63b9a788f584a673bc9b70328a0052f3e62dc477df55e8f4"
+ },
+ "us__scenario_043__payroll_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T23:26:25.548985+00:00",
+ "verdict_sha256": "7a0dda0e9550e268300a5d61435749293eb7e02a7292b1656c7cb28b0efa7340"
+ },
+ "us__scenario_051__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:11:08.050372+00:00",
+ "verdict_sha256": "524f744c91fb09b511e995137a30465794edfd6c09e08517478057c5cff5ec50"
+ },
+ "us__scenario_052__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-08T23:44:18.866127+00:00",
+ "verdict_sha256": "160f92579bdaa75197452ab92dc21c2e2c997933c55dc6ba6ac1f180f84d6da3"
+ },
+ "us__scenario_053__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-08T23:47:41.220740+00:00",
+ "verdict_sha256": "eb6b52636f2920eadab585cbb2b329dc0b8df836b94132d0dcd980ffb28b6f50"
+ },
+ "us__scenario_056__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:35:17.301406+00:00",
+ "verdict_sha256": "ebff25226d847b208498e7808663d70f8fb2540aee3dee16158b53d6810e5dc2"
+ },
+ "us__scenario_056__state_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "reference_engine_defect",
+ "case_failure_subtype": "credit_phaseout",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T11:16:41.534980+00:00",
+ "verdict_sha256": "7f1d11b681e01758782d470ab3c39ce75ea753d0e1d4c31f393c61214f273426"
+ },
+ "us__scenario_057__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:19:45.856457+00:00",
+ "verdict_sha256": "fd1ddc4fdf6d6db672262811721c2125fbbb4c0e42868b2ef849ab25b9c4bfe7"
+ },
+ "us__scenario_057__ssi": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "categorical_eligibility",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:04:41.082723+00:00",
+ "verdict_sha256": "608e3fb3dd7fe97c903bf813803ecb4feaf0a1660ef7b64bac2c3679d4915355"
+ },
+ "us__scenario_064__dependent1_medicaid_eligible": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:06:29.444538+00:00",
+ "verdict_sha256": "72a5dbb23c331bdf620a5b42c79dd1bb66dea050961c9e5655fb3d68aa3c6dfe"
+ },
+ "us__scenario_066__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:15:21.915928+00:00",
+ "verdict_sha256": "2344c626778ff071da05cce666d262d80cae0621b70e0915ee6cc41de04b75cb"
+ },
+ "us__scenario_067__local_income_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-10T06:05:07.402541+00:00",
+ "verdict_sha256": "14c23bf7f38ae94b31c2f01fda8774837e4b599dcd43f38e2d21025c3cf19531"
+ },
+ "us__scenario_074__head_medicare_eligible": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "age_disability",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:22:30.876116+00:00",
+ "verdict_sha256": "e21095233ff1c9c56ba96883ef30dc220c12b6a9082ab80b762fefff10308762"
+ },
+ "us__scenario_076__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T18:44:13.267676+00:00",
+ "verdict_sha256": "8e4680ae3a223014470cb19b74e864522c20ae9e4a3217b126068bb5a868960e"
+ },
+ "us__scenario_077__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:21:03.164972+00:00",
+ "verdict_sha256": "ac25f9a957e6df1a804ffbc0826fb5f27581670aedfe4512f1b136e2654f5e6f"
+ },
+ "us__scenario_078__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:29:00.560849+00:00",
+ "verdict_sha256": "b2d91b793cbe19150b176bbeb6204a4b7809879f36bdfc1042a02906f84bff97"
+ },
+ "us__scenario_080__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "categorical_eligibility",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:34:15.185699+00:00",
+ "verdict_sha256": "00461d72bec0c63e44f74e1119a5a051bd8db0d509dddc95aded9136c2c92cf1"
+ },
+ "us__scenario_081__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:34:39.179419+00:00",
+ "verdict_sha256": "11819fede6b291f15a7feb97469c3975b1ebd32c53cf3d11a697df64deae28c7"
+ },
+ "us__scenario_081__payroll_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:36:05.565616+00:00",
+ "verdict_sha256": "ef2fc03439ff51bc446cee92fabe93e28ebd5e988d31d2173341a7ec0fb8c1d4"
+ },
+ "us__scenario_081__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:37:14.226310+00:00",
+ "verdict_sha256": "434ef3ffad3c4c9e92e1d1a282fd3a08018f1d7f5cf2987227bb60b86a733995"
+ },
+ "us__scenario_082__payroll_tax": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:41:27.145196+00:00",
+ "verdict_sha256": "75e0dc06d14f919cdee0a2e82a8611ea2babb73c2d62d4dbe1d507ccf9e5edf7"
+ },
+ "us__scenario_084__head_medicare_eligible": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "age_disability",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:43:14.074826+00:00",
+ "verdict_sha256": "8a3f550550750e55f2faf3c701a271d21512800edff4229255fd002cb710362c"
+ },
+ "us__scenario_086__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T00:48:12.100662+00:00",
+ "verdict_sha256": "225f0748e81e9e4434bea454fdb99a5b496a67fd9504f477508623759ea52cac"
+ },
+ "us__scenario_091__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T00:54:52.785766+00:00",
+ "verdict_sha256": "b6e8b55b903170f86e341c8a3aee723b1d174d78659fb811d979c404dfd65854"
+ },
+ "us__scenario_093__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:24:55.980556+00:00",
+ "verdict_sha256": "64c3a1b7c1e9dba12bd5460cf18033a610f4f11ed3c685a1977986ef3342c086"
+ },
+ "us__scenario_093__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:27:21.537119+00:00",
+ "verdict_sha256": "3bb92b597392168d3797344d6d533f5469324efe8f9bbcc246eed79e132a000e"
+ },
+ "us__scenario_099__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-10T06:46:49.657279+00:00",
+ "verdict_sha256": "5a96439af716b7d43516665a1bdcc7db7d9b3f72cda484764c67f88daa996314"
+ },
+ "us__scenario_100__federal_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "reference_model_issue_fixed",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T01:05:54.209198+00:00",
+ "verdict_sha256": "a9babaf7cdac3da5f128f36705badf7b3efbaff1f8e94ee9e88886406c5421fa"
+ },
+ "us__scenario_100__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:08:24.120656+00:00",
+ "verdict_sha256": "74f15fab7d138af0040de665895e5ef17bdd031bad67b387495112ceffbfdf55"
+ },
+ "us__scenario_100__state_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:09:55.581168+00:00",
+ "verdict_sha256": "19c2d6d31341942983ad670f4247af19ea823faaaadd7993cc622516b59abe2c"
+ },
+ "us__scenario_108__state_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:16:19.880626+00:00",
+ "verdict_sha256": "52e48492b54957b7ff7b983bc959cb1472a5cdb46a5f115baa78000641a9f88f"
+ },
+ "us__scenario_109__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:17:59.525872+00:00",
+ "verdict_sha256": "9b7a7dbd91b86c0989d9ca17060ed9858a1f75c6f62857245ff6c4cae51e4d0a"
+ },
+ "us__scenario_112__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "categorical_eligibility",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:27:47.216798+00:00",
+ "verdict_sha256": "ac170d3f4cbc670f941a5c58f3cda35f1e513a9e2cd4ac13666f93547a79bb25"
+ },
+ "us__scenario_114__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:29:10.069802+00:00",
+ "verdict_sha256": "1330438b2d1f63b43b908ba931b410f08e1613df5e3ad00c7e8f9202f7d6975f"
+ },
+ "us__scenario_114__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:29:39.683360+00:00",
+ "verdict_sha256": "1324cc2803927a67fa537f5370bcf3873c2261eb9318842bddee4f47469cc0ec"
+ },
+ "us__scenario_117__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:33:34.425206+00:00",
+ "verdict_sha256": "3162677f98080f94f6cb0728beb1dec62f5d1db1c2e2802abff91cf51414a2e2"
+ },
+ "us__scenario_118__snap": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:39:10.605620+00:00",
+ "verdict_sha256": "9e3071993def3b401623d836b3c5b9f372ab908d7dc02a1c5a5d32db88d5d858"
+ },
+ "us__scenario_118__state_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T01:41:10.126431+00:00",
+ "verdict_sha256": "3f4e0f7e3d6625e5e4632e0e16d77393e6e264c58b8aa5511a853fc5ec2055f4"
+ },
+ "us__scenario_119__federal_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "credit_phaseout",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T01:44:25.406626+00:00",
+ "verdict_sha256": "cfa26c1a66e81f4b3bcb62267481ecd63558efd4d9d79e3937e53b9096828cd5"
+ },
+ "us__scenario_119__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": true,
+ "judged_at_utc": "2026-10-09T01:47:30.156708+00:00",
+ "verdict_sha256": "ec826f2d7459dde71206fc346cc4f3cac0c54c68e2bb00297312d56f39b2a444"
+ },
+ "us__scenario_120__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T01:47:46.213458+00:00",
+ "verdict_sha256": "85b90ac023b424b1ded692bdfcf1264c3057abec7b76a9a9f5e969be2c1d7f1a"
+ },
+ "us__scenario_123__federal_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:30:57.145495+00:00",
+ "verdict_sha256": "4f6f31d6328752b8321dd1bf304c4fc915568398d887aef734e362c41ac99e02"
+ },
+ "us__scenario_123__state_income_tax_before_refundable_credits": {
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:32:09.911176+00:00",
+ "verdict_sha256": "ad43142d632889e7c01321c24189a6c1c30fac9148f5d24c28123cb66fa7b12e"
+ }
+ }
+}
diff --git a/docs/haiku55/seed_digest.csv b/docs/haiku55/seed_digest.csv
new file mode 100644
index 00000000..31e50c9f
--- /dev/null
+++ b/docs/haiku55/seed_digest.csv
@@ -0,0 +1,675 @@
+case_id,prompt_sha256,verdict_sha256
+us__scenario_000__federal_income_tax_before_refundable_credits,667c9c1561a5fc3f0550f4935b2dd825ef6344f7e96e3cd7aaa4a38b604bbb68,843a19da6dca70f2b9128f9a0c53bb557e281ebedb0a8ca5647c54d2f84950d2
+us__scenario_000__head_medicaid_eligible,8089c5b7a0466e36ef70a540a36e0f4d2ef3e03016761d5af347bdf3413c3169,23da81291d2f22a85339bb188697a3ce2973dabd3bcdcc2c9542da0e89f44968
+us__scenario_000__payroll_tax,ad3ec2acf8cf3f51cc1b1163b7235de86faf8155ae40414008d923bd19fd4f68,2976fd7ab6bd851cfb4280f8e4bf6237fc50c8fc6bc9c0bb5fa1bef7ea5081ec
+us__scenario_001__federal_income_tax_before_refundable_credits,8e481d2423d8a00d78610db517ca72ebe146b6cdda3bede219de8a605f1537ad,26332b1907accd4a10afa346bde68e6f17f61627dbcb7a66de117a3f3e78fe98
+us__scenario_001__head_medicaid_eligible,5b02a3162ff896f56d86345134b7eaedf86a92df2dc4385cfd12c4a8eea72072,01d80f20bff3223c3e6e45fc067fde2a8dacd0735cae5e387bdc5cf010128119
+us__scenario_001__payroll_tax,9823daa04787348f26f579dc51f57398cfde0e8ac182b9b7b4039777536b61ea,73dcfb1b3d84dd5872bc6df2284173bd61036e6a87f68967029a4c7c8bf798bc
+us__scenario_001__snap,194a3313e5d135c9a819556944a0f5e62e7bffc7237f6028662fbd2ce2cc0593,3c28eb95c4f073cec7b6b557eb113f8e38f378e353d2767e41482c606a729613
+us__scenario_001__spouse_medicaid_eligible,0e4e93d6427a4d287d6a94d667ee53a271844c4040c62a1d59280cece0a87d53,881bc970e58f5c834264dd8cd1aef813e3b88f4288ac360c4530a2a60fced54b
+us__scenario_001__spouse_medicare_eligible,0656f728338046a43a0d834c0f4142b45f0c2769fbd28ebb16e748773a99e1c5,cc4bf2ef991e1d2df6eb2dc0832bea10d4e98313a634a03c95650062870c9b1c
+us__scenario_001__state_income_tax_before_refundable_credits,ec40241c45e8bfa855423975eb9895d04402eef36695e482c368d92d75b075e7,76e16fef6cedb66eaddee61f05552391deed824c858df115640cb34b9fa08e91
+us__scenario_002__federal_income_tax_before_refundable_credits,3879086195ef09bc9fbbb4224f27c4b266087792861bfe60dae6d4ef829be7f9,7b4c64608ce0152998d61f834b2f56e051899f304f1e234c6bc8d67f31d3a53c
+us__scenario_002__payroll_tax,8e44f4b6852ea58a9cdbd8f3c0dfde061fec4277853883bcad021cfb08f86393,6941c2013a9ff6a340bbdb01caa5d540ebbd16131ec51ff8c5c1a4191e37e078
+us__scenario_002__spouse_medicaid_eligible,f31870852cd979c20818b8eb95109f28f97bc5884f730d8bb088eb1632540437,0614261cf2a4ae6b08880ad85cba80f2c8422e1d427e790c0af98547fabb430a
+us__scenario_003__federal_income_tax_before_refundable_credits,ccb734da3c4f8b8880a82808698e692359935925c1851d95648b6e8d8b0907e5,bc4c7f9530a23599c78db9c68c70d94a38576dd4863fcbc498a1718019e05da6
+us__scenario_003__head_medicare_eligible,3d383853ed8d2408b42b2d8f860e8ef13d837147fd4aeb7a79a2bd2767d65876,963de2b07086fa8200ffc7044fae0f2814b6dcf4416ece7f6251ee761b1093e0
+us__scenario_003__payroll_tax,7bbbfaf9883624c09846b24f41b05c655297f7909657a7b6338b59eff910c970,cc8373e601995d8be14e9a3bde16c2eb9782dd7b5b4be7b5b56ffd550e4c64f6
+us__scenario_003__state_income_tax_before_refundable_credits,887a8e3537b86044d115cf9871eee3fc94fe0d657c12db4fb74d54ef84754d4b,dadce1fd2cf0db15a840b85aec6bd3eae1f9ae0f90dde245e3fb8f352cbf8e6d
+us__scenario_004__federal_income_tax_before_refundable_credits,9e983ac85d363eb014b702bb595f1d7c7244ea1e86e64cfb065f486ce116e9ea,594e83cb33e224a96cf4164e8d41c5683f2bc0c1c0e9f44d9e5bed2700b3c35d
+us__scenario_004__head_medicaid_eligible,2675aa52eb07eafe4f531ab52c6d097e3d9bad454cd1576d5ff7299fd6efd176,a3bf397cbcc6f02c37862addb0c34de649b1b18ca8ab2c862ce302a9411ecbe9
+us__scenario_004__head_medicare_eligible,c4741b2164e29719d5b32819b5973680169dc4b65a5eec64ac5fddb8df059e27,4c97ce28067f3cef1da48d7d4b1f429c3dae32de6913fb5a27429609897aa84a
+us__scenario_004__payroll_tax,6564a474d536aec4c4493b871c44efd8d7cb1bbbb8d8b40dd8e8b20e95bc28f4,3a5985011267f36bcb9d0587e7b81a3c5849723e80eba716a88038e2fe5b02f3
+us__scenario_004__spouse_medicaid_eligible,dea74d788c0be58164bc9e89db06eafc48b2ef87a41487949db5004b53aed483,f67c8c87218256a0f55d4ff186e8c9e3e00ed87c5486c0cdcc84ba779e18f4dd
+us__scenario_004__state_income_tax_before_refundable_credits,79e3c8505e4bf3b830175a7a82e1f86c3a30860d1139f78cbfdd7525c16dce73,ac22060d7debccb45bf71e79e45df9e54a606222c64945eb1abfb097b3696605
+us__scenario_004__state_refundable_credits,e65faf7c6f5dc4487aa71a232f55f12658775f0b65e6046a8721abbb20228a35,b1b6c3aea2d345d1780d1fa45fe90a705323880794e00ea121d3951b013d12d7
+us__scenario_005__federal_income_tax_before_refundable_credits,7aa358111f92f9bdb00a251f74905eed494e06b4e569436f3d2d6ae34fb24895,84cd048f4c9bf573b2af150d703b43681a5016b5c7d8ddc157335364490bd51b
+us__scenario_005__payroll_tax,ad4ec8ed35a8691cc6df8a956ff8a92892075cad3d1ae0f9b3bde06c3942462b,97fe9314c0547c080ea3a24bd4f6ef3c7e26dc484a7cf3be1216f512387705bd
+us__scenario_005__state_income_tax_before_refundable_credits,962c40e2d29e31772f51cb70a90db6d09a027e209f781a70d77bc84d3ede161e,428c7dcd0356d5b08b0d42c5014cf50b7e113998eb446c9249b1f9592d556cb0
+us__scenario_007__federal_income_tax_before_refundable_credits,ae7d198f02cbdfbddc6122792fcc78c8780281ba2185cc43e4e5487a88af9451,ae4f45e4c06169ac8062ba546969e25a0e2e25569b56143c4811d70c95867032
+us__scenario_007__head_medicare_eligible,d6f3575f5ba18e234272f55c6dd9f63f19fb7eebf4e13c0647b368d8ca4518f4,3f881ff75efd3f35dfa847af2c94cb597a6fff67eaf70f139a3eddd9a9faaf24
+us__scenario_007__payroll_tax,582c724c86cdf328d26aeed0be2f7982255366d6f05bdc22aead1ff6f68a1565,e9746dc99fcb16bd82cfdc51b35bd027e9cca84a5e27da88fa8b68811c8e20c5
+us__scenario_007__state_income_tax_before_refundable_credits,769ce0c49e2269b1cf68e955ef21745daba7e8f9a7a2a6b87ca31d5330050b39,a8ee7e56b6b6ce93754d898ee40286cf513e4bd3518abb39b3c2e4545150a1cc
+us__scenario_007__state_refundable_credits,f499e91961a2fe504defeb2aacf35971dd388808aba35c9581d52c6a54be5d3c,ef3ef8ea2b037bbf844e47e9020d32e33e9996fd247c00d7e38f2997300bdffa
+us__scenario_008__child1_chip_eligible,a93978d5d293cfe92935b099a148d277e0c77be2ae086cb1d20efc27cf3ab30e,507b44d055f2623db2791e3a0b97195fa6353e019d482fbf7b26ff17a055d83e
+us__scenario_008__child1_medicaid_eligible,546e03b9768c3db88d457ace694fee63a5d0e8d71e8fc46cf8871a619fa28362,063cbc6c13a76d76c39c448a742b7ba7f58a012d0fffa5a2feff6a5a8055bae7
+us__scenario_008__child1_wic_eligible,3de7603916921d46a583fb60f793984c2a22a885565ee7b9995d7a51c4710dab,5b9e51ff86cec1b37a9a4a1e1f2ca5d7a21b0ac1f4e5fd88ee6b06f789c8e4f2
+us__scenario_008__child2_chip_eligible,a32841e960466ea34fdc40a542b6aca12c991e6f47eed26748eb0953060a92fd,e36890a740d48f7e0f7ee115c17ecb461eaeec02547a792bec190b04885b00b1
+us__scenario_008__child2_medicaid_eligible,a9d876d93eb0a849ecf0c811c87af8ddbc6c1d819e1734eb538ba278d050e4d8,ed47b0ccb8d73d15468005c0e4119960a42faeaf74395c1f270b8b2f3d2b8906
+us__scenario_008__child2_wic_eligible,7a64b827ebe7754de190f98fcb49b3f5a8a86f4f658d603158d798fa9da84953,be6d060fec347cdf4302a2a366d0c89c94715a71a130da165e20b4a1dca9b73b
+us__scenario_008__child3_chip_eligible,86f3d921a868bd221ad58746c9bf9f784b67aa30a3135a5a15a42964e4aff436,da787274666b5a020e7c15795d7beef35c29ea2eca43548681504610c455f36d
+us__scenario_008__child3_medicaid_eligible,e4cdde65ea61f1e456b8a3b6ef4a8854d2f0b5ae3204f96fe002cca5c9ad3308,3c186a055604dda9bef0aec3a755bdd5eddb762baf246c3e5333430655580971
+us__scenario_008__child3_wic_eligible,91e0ae66cdd124b41e272ae84249cf23d4c6e226d1d7c068e598c551854c8dc0,6d77d109d3db2b657e9487065c82d1dc7b9ef1f30fa36c1ce019188a8e6ab6cd
+us__scenario_008__child4_chip_eligible,f72543fe12a4a787dbd68130c3f0300a412f304360666158e5b69c3dcd3acb97,6848564d2585fef5fe7c09011161ffa2b111b7232b44ee054e6b0233157c5783
+us__scenario_008__child4_head_start_eligible,d05799d4149aee75cd136aabe88031f8729e9373db8d95128c207fb33302d947,4919b713b0fc80cb5c1f20bf7e75fcaf3116a0ebf57160ef7c5c6a892db8d5e2
+us__scenario_008__child4_medicaid_eligible,0d7638d57eef642f995ebc42128d99a3d526c2fb5b62de7e6c36d7b4a4078568,9707bca5e6b5e62d675fbbdbeaee6f545bc98d1d8a2bf5ad11d68dd7bdd10096
+us__scenario_008__child4_wic_eligible,81e93b029af16ea8b29b314c86ee7b821b49526c8258fe30a443ac42f71c49e2,326da5c1151f6af665504bb58aaeb5479ca5eeeca760d9a5af3aa713fa304726
+us__scenario_008__child5_chip_eligible,937edc18fe50c08bf1d28180cc7ec4f40e48de815ec078e311d3bcebace8d8aa,2fe026fd635afdf82c4519a1fad61ff70318938c1556806dfe9a605a37eede21
+us__scenario_008__child5_early_head_start_eligible,9f3b2d8d993601c7bbd21fbfefc51f64802eff6e2d6ff21f25d62cce81904de9,97a7d39317ff16aab8399cfe5826cf591941446ea1cf32959b18ce7b6983b23e
+us__scenario_008__child5_head_start_eligible,575909b03776d949906e51d3c4f1d60f5b9c50ea316a7eda58ba6a04736a33bf,3132e5bed190bf98ed937baf394957bdea0d024e61ab73e51d222776495357d8
+us__scenario_008__child5_medicaid_eligible,540ef0b5902d898040af3718b670763aae644f2b1ba6bd11c9fe2ec0fc805f28,8868cb9513f817d2d8319f1222377466c24625cb8b7cfa8fba1a48586e437f12
+us__scenario_008__child5_wic_eligible,3793bf4f76035343b5f6455926a085331224243c83afc023002f34349b7a9c7a,cf03b90d080cf9199b8c535d03f855a3b95778a1085ab6e8a9681b00fec6684e
+us__scenario_008__child6_chip_eligible,6469dd1346fe7dcd5974bdadba4a8eb9317f6b59de0380ec7a6c58a3128914bb,f4def16caae5f96f71aad03016bf6714159193c57a8d8118793c6ca471b92e61
+us__scenario_008__child6_early_head_start_eligible,b8cf0090ac647bf2b64c32d327415b9282dd118091be384cd510a4e08127af37,81d424fff9979f86a65e7899662b85a2a281ec99e30ac89a6f0a6f5f5a945e25
+us__scenario_008__child6_head_start_eligible,c58dab1204d004755cbd5409a689fdeb19f54eace9683eeebfb351393d0e4faa,e10cdc003427fc03bc1710101cb9cb9f49ee34573816e2e308a8e3ffc45560f4
+us__scenario_008__child6_medicaid_eligible,dc0b70b2df531cb004115d50f027a45358df298499f674d74e02c5b93d70a189,214ab2db89a15a0b69f5ae2f2cf058eca9352833485400cd712cc8c02b7c04b8
+us__scenario_008__federal_refundable_credits,4c6e54d2aa385237512ffaef1614191922632db480db1a5c548dbc99eac05a24,c3aa5b8ed5396186aa25ba8c6acde571d684f35a128375bea0a6a17bf0e91020
+us__scenario_008__free_school_meals_eligible,62f64561aadb71fd2cfda7695d216c0625e7ab435dbdff4afd0f12d0c353c18a,92a815206251bb5249e49a462eaf21ff1e7fb3e1fbaae01a2b0841def2be9ce5
+us__scenario_008__head_medicaid_eligible,e7a663ed314f9e402e824eee6ebc20ae9d3f102bab57545d2c4568c03b65fd26,b8643c04ae58048b8d04654ac041fec217fa3e2283947e100481345d640169e5
+us__scenario_008__head_wic_eligible,92a59dacf6ec33fb005b24c9ab503994ce3de96e9a18ee76da0fcbddd80dd2cb,abbf4b1492cca1ee52c3966140730e888cd04342abbf4a450b4eb922e0487ee8
+us__scenario_008__payroll_tax,0a4085dd94512b12b7f114551cc5979c87a8ef12a13e699a715b5c6aee11e111,b2c839276378cd99aa7ae9088634d410de0956cfad24d0adf679fff6fdeddc37
+us__scenario_008__reduced_price_school_meals_eligible,52c1c6698450f26671ddf91fa3d5e857c24126574d979402db2f7e2ad1003d14,5f7a30df6cf4fd64e856bcb5599c16260ccdc2938b2eb70aea40e24ecf26cdfd
+us__scenario_008__self_employment_tax,9f4e19d631dde986e607c4d58bc589636aea04f388687cfd5cd00989ddc7a5a7,5b669021e8ce7cf60f9bc250336a693d97335cb229db0940c3b6746c2c6f1317
+us__scenario_008__snap,25bc5d2714e1ee2df11ffeaf2a70499955a4dd80f029b920f1ef6c7fd9840758,940009b70fb3756de36e8235e75247b79b3bba7eb235ec55a70ff00929615e5c
+us__scenario_008__spouse_medicaid_eligible,cbd76a85e0b395b002e0b988ecd19d3ad38d77b61f5c0df5bd30395b67450e2b,19d57f0fee5b98fbd5dc3642ca577b3459247c8b6e59164b8f05edbbb86b1c96
+us__scenario_008__spouse_medicare_eligible,632f38d7eefb215099c3582020a75ee22ce5df9fa329beca5da6d25e466ebf9a,56784f5ffca023f279575926bf1fe365c6d6544dd1a60487622e021954c5ae58
+us__scenario_008__spouse_wic_eligible,bbf88b0edad4103bcda7ffbdce9c4859b9ba0aa763555a5567232c68ef0a8d1e,0b5455a24e0ada992eebb21a33887b7a38f5d0001c7a424fdb7a54ca9a0b70a2
+us__scenario_008__state_income_tax_before_refundable_credits,bd12f82c481544cd55366a10b31020434dba112c66036aa48dfcdd8acf9c0848,a8019a1a48879df8d9dc9e7b45f685bd38e40aba112a9b250e9980706eee7038
+us__scenario_008__state_refundable_credits,367335bf65391125f25e867c5bfb9735aef5537673944694d3e139655338c4f0,445d95638a10c09500d4b5dadfc2d835f373b0954b5ebf8fc82ee49f4cf53d29
+us__scenario_008__tanf,64cd17572bacb38973967f48da745366486b8d8f5bae613b2c8a9650e3ef1b72,c91c68be0bc16580d986730f2486ead82472c8e4e98ea1c38630818502f9e26b
+us__scenario_009__federal_income_tax_before_refundable_credits,c5937b4eff7a44ec14588200548abd350f892db43f81b53405c037db1c340a78,2a0b953a00569dc225a1d912e1403bbf0878fb0a3754ee9bfd6fcea06b1e4fb7
+us__scenario_009__head_medicare_eligible,405b568c55ff1c1e31824d4b948ff80fb00ab44d4d02bfa7bc9285abe703f8e9,58a46ae33fd1e902d2c8ce0f93f92f0cba6bb5a0b6061f650cd369afbe2b52c0
+us__scenario_009__payroll_tax,1f3919bdf51bef4cb853f4dce02d98f9b0c4352c0f4f6ee76dd734874e904c2d,93d539ce4cd8ac53c946906c761024e33ad5a3efefc92e0c733f6ab46fde2a99
+us__scenario_009__spouse_medicare_eligible,25c502992c8932a7eb2112bd9d8caa000a82743a864544c26f036ff7e39dac40,76a2cffee43533b871baa2bb6353e2322c5e64a16f255e21bb79c2f226faa4d0
+us__scenario_009__state_income_tax_before_refundable_credits,a5d03d9720496496b60ae0fa7902a5ad97c1e8d7b429cfc2dba1140345b5609d,5aca2be5756bda2475f6dd57a58925122f424596024dd6fe2047a254cf0c74fe
+us__scenario_012__child1_chip_eligible,084057af958a4122860d69dd7a951bb0fa21ef72cb2eac1334af0436eee0445f,6559dfd679755953b1f54d17d541830275545647f4a3dfe62650d1f7ea9fa637
+us__scenario_012__child1_medicaid_eligible,ba610b0ddf90239d0506340cfb77196e3a36e761ee9e58727543ccfa3131fca0,61ae0f73c61c3aaa837136b616be17427bf2aed3a29231f1f814771f5033f2d0
+us__scenario_012__child1_wic_eligible,c92b228c812ff8a80bbcaa7e1aeeb00e01f7e3d9cfc0f9fd704bf0bc0ce30151,30db9b9fc4aab1a5b44ca7204368ea86ffe019ecd68123b1e6403d0277093b57
+us__scenario_012__federal_refundable_credits,5e71531c0c942e6870f1152fd37656bdbbdec3d94eb993da7c8276d0a4bf18d9,3b3ef440303b4536320b363f37b3c4447dbb39cd9709f4de9d46b6801eb387fc
+us__scenario_012__free_school_meals_eligible,cbbc6345ac9b2b342390f2b4ad67f5f05b6d44521cccf3fa0eb236cbdf72a995,15519c0204fc12014361755deca6b28cdb67e7877080f30fff1667df0ffbcbb6
+us__scenario_012__head_medicaid_eligible,977259a2fc27fa7804e31e8cd2a0d1f7fc5f8bdf9db3b81e92e5b80bdc6839a4,246f3b374a462a06c56a51215437e56d2488d088f1c94388960eab2c4bb934df
+us__scenario_012__head_wic_eligible,bcf0075d7a75af15a600f90b50001c63f2d3b1b9998e4c7cd199b837c9051fdd,a67b5f97188d9462b41ed691b75bc934ad1ae9c9aa2d1e05895e9c8cdd8374ed
+us__scenario_012__reduced_price_school_meals_eligible,57e51af61f9eedf0319eb1c6e6e5765ed9eb8c48d9e295b64e4ed1fd6f1c47be,47d52e1d7e6d98130880554c72148d759b8c81e87cc4222fab9a06e9c8e65532
+us__scenario_012__snap,48f650789fbd5ac179195bb1d19992ad755fe513a1f6e563503cb65ac1864117,e3296ea51701bbe735d6b45cf7945e74c747917095ed3195cafdf0fdc845e28a
+us__scenario_012__spouse_chip_eligible,ca2e2b03a2a77572e92a11562373838658ce4c42745d2f6b0574f6854c4195f8,cc3202be5f93f24570bab75f89f47c84f238bad90eb11ab86edfdfc40e1550d9
+us__scenario_012__spouse_medicaid_eligible,381bc83ff39fe00845ef98bbc58821e897b2916469e4f310dd86e093d6d93f69,766adafd08df7490d7469ea1ca4e8bdd712f707a5982508777c47c7a29a0e02c
+us__scenario_012__spouse_wic_eligible,26c85250889e2c33332d350fb331cac26d5213d63ea806e88dbf0d4b5e30ce13,90ae25f3122f8f48fcace59e58f4d08acf154763165613ced5520f1725cb6039
+us__scenario_012__state_income_tax_before_refundable_credits,2bb19b10949c5f34f308c3296d315be57bda75ae184c5a56bc9efdcb5758327a,f946db2691209ec8a6ecfb5e5c92f763adfd9cc1b1a04651036bb7ec20eb77e0
+us__scenario_013__federal_income_tax_before_refundable_credits,49b5758bc0ba32cf35934c72284f487c90039703c9fb9c78c65e6ff6c0b2e59f,9f8c1f874fb0670067b450356b5b6040463d98ee3ccb9c6c4cb1465fb71c3bce
+us__scenario_013__head_medicaid_eligible,46f55f7fd62870bc3503f9b6fe081e3be1ca4c1cf9a3d60d1d2e8c678aae8aa5,5a252dfc9466fd48248f6e8fee88e41918945798efcd6f781d7d9473e5f529ef
+us__scenario_013__snap,877a255b81bf6ccfc13dd8fa884f8ec405acb9fdfabedca84b432ba557a12623,52e7febb6cc3645de417610233309a9844b501a851088dbcd10892fa827218e9
+us__scenario_013__state_income_tax_before_refundable_credits,2b20bd6903659831c0640288c3a9df49286aebfa24220f5d3df6e5b86a73b539,df10fe12419c9927b1d23757b7fda20bb6debd2ef892504ea1aa3ffd08f0652c
+us__scenario_013__state_refundable_credits,255ca99d6e7f1d7650bb9283eac1ff87b231dbb3b28530004f6c94c45b7ecc38,395cc101c368b10226f9c650a938608c6cf12d7c9b0f4a8ba14cf2a74bf2a652
+us__scenario_014__federal_income_tax_before_refundable_credits,f5d13275e21b97e6556e558666c6eff4b14b45143321da369c355475a37092aa,f4781501411253ad902d58fc6a22a8680defb8d94a83f909fde9d481e81ae297
+us__scenario_014__federal_refundable_credits,77d73d79de6d24641da717e264acaa4bdefe11ad22f59daf6d9b1ac6015162f3,ad32821e5adc7fcf36c1d1587bf10514d6dbd6195cf48a5945605a22db528f64
+us__scenario_014__head_medicaid_eligible,313027647d0287cb666cd7e527d232801b09a99e676b451760ae3008bb6c28a4,5ea332fe1cf93c010ce785127c00d0eccd326b79d0e15b1d7bed20f7905e9432
+us__scenario_014__payroll_tax,69a26f3b5f4c84b500f982be577d094ebeb46eff9c18e7d54d7be54790e29f5b,d659eefae4b3e438c7134dc28d00ca2e5148e39b25a59af7dc25189fd9ca785a
+us__scenario_014__state_income_tax_before_refundable_credits,6a26c76b208e8052fa8e69c3128ae4b3550da81c346a063662cfa4dc7d733084,35893721042191dbbe02b9c066d5c0cbd8a8d8b28ffed051bd8e346b35713db8
+us__scenario_015__federal_income_tax_before_refundable_credits,4e4fdb40b8a3155b266f63ed0d99b7c51a04150c028fd2cef2ba9f0d6a6bdd06,79e42186c82e3feb6befd89ec090d3bb29ec13a79af4e44a9bd504b5db1c4983
+us__scenario_015__federal_refundable_credits,9762bfd026cc4e732c792f3b5f0715c2b19cc73284c034b388d311fc3d2b24cd,20abcd0302b072f8fb64b41f73a46de45b9ae1c0e56f131e142f11a7a78037b2
+us__scenario_015__head_chip_eligible,d7ceb52a5d6378b9178c11b36e22f119ed4622ee673b23c0e55ec64114fdf9b3,b2bfa22e6876c72ccaebc38f904d41cbee831b190c61438ab9a8fa8f2803ba43
+us__scenario_015__head_medicaid_eligible,16802b03cfffc8cd49b6414abad9bb1000f2797a76ffe6e6bb67b8c0473cd134,5a389e3f4e8f93e1caabc888cded222f52155a41df43eebdcf41f7b087a04d34
+us__scenario_015__head_medicare_eligible,5febd3eba3ef6502f1a5c9d44669b319d529e83e3979c3a1b6a053904ef43953,64f9bb5d5d53143e09d031c5050319e5011937c6679d0f4ea3acfcc482483c37
+us__scenario_015__payroll_tax,6a77bb32e0efe4f1fe6c73ab04dcc5a94066b37b2a5453a0fb604b8eb260ee26,ae846d93a2d37143ebb4bdd6ada238f55427876a921e626fccafcdb72a37cdf6
+us__scenario_015__state_income_tax_before_refundable_credits,37c1f9b5c7ee8f9dc8250bedd8296a1d8d76e299235fb6eb4bdfa86c4e17aec1,1b92d5310bbe3563809c2c0fa680ef7706e8a927bec11ada1562aa8e5200be4d
+us__scenario_016__federal_income_tax_before_refundable_credits,0fc24268d6629a55279229096707458ae759f7da54d6ac37dbc00e4df915bbe2,d12b9af1b9423654b1eb2f16e1378a37ca6a937e8d13c2fc970bdfa0560e6df8
+us__scenario_016__federal_refundable_credits,74c125d608580696a0127c7ed959c60eeb94c73e5124cd773124b191b308745f,753066bf7ef3c17ab8edb6d9371ac9031683926746432636369e9874feb389ee
+us__scenario_016__head_medicaid_eligible,34f8bfcf0c92652d3d292df745e1155d766e01eb782a35ed6599aa7362aad3a2,b87a14633b6c862b2fb1ec6a7294353d6528aff896cd47c55746bf27d893459d
+us__scenario_016__payroll_tax,a1a09a78f901c8a7ddd61c5e85d56187e7b9ef17e1ea31d093a8d7452d903c31,6de254b0730aaba0aeca8fe7055a17dd0d255c0897a658d03251fc983053d506
+us__scenario_018__federal_income_tax_before_refundable_credits,8195d0c0ada006573bd5fbe15fdb1219bae3a6881340699d9ea586ef0701ecef,725680d89160d57b412219dbc5da326d8bfe33a044c30f818d9472618c353433
+us__scenario_018__payroll_tax,6a3ea93b3489138dd01b6be096119b9a3f0602c388f5b3d927ca33a356e96de5,dd07f311882f306f06fcec6cd99e9358edce470f5b1f41e470f9c071b53a2047
+us__scenario_018__state_income_tax_before_refundable_credits,14d1178cd746f0e92aeba05ff9b50c7c5eaf4c706c8bf39f1fc7ec9963ba1022,6c58cd221a4d473a5c77b366f3e6ea97df16ed45572ccc5a25b6adbe256c8008
+us__scenario_020__federal_income_tax_before_refundable_credits,485af70bc42609e385f31605477b47cd0be67368eea69d0b261652a42cb36535,9c178d5126f9d8df1a74ed4ee666b51223b268dbdb4eaee6fbe918e559ed41ca
+us__scenario_020__payroll_tax,f3366d48e05a89d198c6a931766af662a36245c1c688a83ab26019af95a19371,95295e8e51c7bcab0f7f19c37eb9fdbe504d6dc13e1b2646e17b7683eb7890f5
+us__scenario_020__state_income_tax_before_refundable_credits,6428e2ca34286ce588df84f3c842d5e27e9d214be9ecf690ddfeec7eaf97344f,dd784726d5743c8a7967fb027d6088c061f2389ce762e0ff493c967ee3aaf445
+us__scenario_021__head_medicaid_eligible,5cd32520de73b23fe84a1ba52d95619d48577448e2dc2a52f8fddf992e02bcd8,a7579832fd243cc634b16175ceee8c925c39a09867d89457ec3d00ccf93d57b8
+us__scenario_021__payroll_tax,e3ae3bede8198ce8a7d1360392a0835e22bf9283e8ef0a31e6c3ce7395c868bb,7ae9f9a54cea4a2f6d92063344bd6132c8d7feae82cedd64a14875b2ca0b1518
+us__scenario_021__snap,835e04d088d82c85644fdaf4b0324ccda77fd258e69759f3ca916570d6feefd6,981799807d4c70f1c827aa4f94292b61a3cf6f1d7a4142ea19557d9557aee098
+us__scenario_021__state_income_tax_before_refundable_credits,5916cd9b6ae77aa81185ad1717dd28bb20835e4c216482dc5a05a661581e9c0d,d117348f288ef0fe5daf6fdf3e0eb4f2b620a83dff20859874171a8c3c4f4674
+us__scenario_022__federal_income_tax_before_refundable_credits,b480d10c33435b168bb0f830b76eb3197a3d9838a47559ca3bafbc83e7962c03,60720f64eb01fba3f686a353daa954df2f1495bd1e57364ca04288941d8613f6
+us__scenario_022__head_medicaid_eligible,35a31d563430c21f8372ffa8490df7cf4d85bc4cbbad8175355aeefc62960bc4,c5beefb6218d89d0f2221698b4cdf2eb07f7237aa5204620eff8e71a43cd9e9b
+us__scenario_022__payroll_tax,8e74f195331a30b9b84330efae8482a2bcc6ab9834de380390c485f8bd0a84da,2139ffdb06d808e00873f5df1863543df45bed7e50287a42ca26c7aabd28b12f
+us__scenario_022__state_income_tax_before_refundable_credits,5dfc64291c367e7604afa9b27c5c092d15a762ae68685f8b03ec56c0d6be6ea0,94989a808fdcd96c8048a081fdc3e97b7ba9b69dc11f92c8b20f760ccc5c70a1
+us__scenario_023__federal_income_tax_before_refundable_credits,24fe86668754f4da309b7adc53da6c4245feff9fa021c5fd707385dfd503402f,29605b43f6061ec55982a4aadd99bfc2b6f1a098e2323ca2aa5def74afa4d564
+us__scenario_023__federal_refundable_credits,fd3a3004e6fc4808b6cda239581f0631439e72164c16b8e3655de63ba34c621d,8e9052d3617bfd55244b74954371185d36d30b8be607f8ce5e7a2d442af300da
+us__scenario_023__free_school_meals_eligible,28e17e05d39de7b8957cd7237eadab9663714555febee49423c59fc3acd48a48,ed682d23132b5d3cab55b174133edd278e9e69116ded256002ff75b0fe508bbf
+us__scenario_023__head_chip_eligible,7e5a7ae0577899811db282d1194d037bdcc6d1f3bfbc787d590df0977de38bbc,7e2ead8b1a61e568e5df9a5520e0cbfb554fe62d32db0e0548f5ebc4e0095edc
+us__scenario_023__head_medicaid_eligible,339d113a4162a995e6e9f271ec2c6656ec2dca8c6d33f18225bc9f8b6dbc81b9,5094b4eda857c36ce42cf847b76eee7ea7f082c61eb623fabbd7c2e324b28fce
+us__scenario_023__head_medicare_eligible,327370c1f121b297f87d80f70a5c68b886989d329ac6e15e53089e57c29f56e6,fea3f5ab50a0314400d016ef46321a908942e85831e8f6ac53af10427e2202a5
+us__scenario_023__payroll_tax,46ed4b323d96a057768cf17be38c9db67c43be779a3b6537e64e1903ec7995f4,99843b1f8680e16165a913cc1e5e566cee105f883a33a7aaa0cd9b2f9660bbd9
+us__scenario_023__snap,e39a7acb2018cda34ae17e1ba7fd1f5459cdba50daff1014a5fe3dcc640895ef,2324dd2325f39e80116ed2283974cbb7887d23de86e23933d84b4ebcc7cad5c9
+us__scenario_023__ssi,18155fe58781b257180794ae8d40ea3f566eda49c11ef9a9139128b21b6b0e61,c238e5f6f464eb241c29970e3ed89d9b5f0364be2889675d2365cd62e51d3c21
+us__scenario_023__state_income_tax_before_refundable_credits,e5e6e9149de3528bc60f40ff578df43ea80ab5eae13dba2b1f4769c08d1922f5,961e2741c24ff5afb00508075290d35aa94c8d292cf88c41939918fcbbbecbcf
+us__scenario_023__state_refundable_credits,959f531b17d5f4440bd5effcadcb4a569373157d2000edc79e71c6fd85426a38,ae3637ef365efd857b5aa2cca12b42b7d0e0845d91ea3dcda25971189477ff6f
+us__scenario_025__federal_income_tax_before_refundable_credits,c8b40222f42254d8625417cb28057568ceabf08ff18c8faff2be8b8c2bb5bb2c,d03fbceaa10b0e45b6fc667634cdf57d7b4224b848dd873a4e399dd19a6b13d6
+us__scenario_025__head_medicare_eligible,0cf47f2696153fcb224785652ba40de7ae34723671fe6f6e45582e5b80cf3aa7,9614ddd0f70cd0a6f0a6a38561f78ad12d4e07e9f99e0b8039ba2a12dd1440d9
+us__scenario_025__payroll_tax,bb0ca83f76a0e03a92c58a47c4c598ccfbeedc86ae889287ece6123466943ad4,b5ad297fb2ab2419cb15bab678129e12fd348619bf88fdff45946db79547014e
+us__scenario_025__state_income_tax_before_refundable_credits,d265cccd89eeebc5eb02cbb20588d3e04a6412c13eeec59b04ed2a492a68de65,01bf35913d188cb3e5793cb4de5a001dcf05020ccb54b1950e63c9f69d4af10a
+us__scenario_026__child1_chip_eligible,61624d00a6f374c5e4ab4411533191b0be374729ccc3588f011f733ac172c20e,07c907416362ff6b3844b9f1a27e7d24e7b3234982e0d1014d4d68dfca39c406
+us__scenario_026__child1_medicaid_eligible,518cffa19c82cdf94ffdf4eccd2948c70f1184e9d19ef6b7b28a4fae26bf6f01,62ffafa30630a49c589b92e569409371ae833578ad2693994d92ae1506a31a89
+us__scenario_026__child2_chip_eligible,16a8827d04f3b31e2402671cc1f1d083168910f998c11b3372186a663f999d19,a7678031d55bb7f2bf139402c1151f1afb4adce191b7191de1a4df7ba7985c9e
+us__scenario_026__child2_medicaid_eligible,453cc47e30d5b1e8d320a0836f10c5eeb966c88393020ab07dbf418dc80248f9,a948872af44bc921d9705a5662a7b809d0b6c151cb81bee00d0dd7f11c9dd988
+us__scenario_026__child3_chip_eligible,4f5a3b1236a67a6675c7ab52fe4b27a847a94eb3fdaf03bd33ffad15e93420a3,145e9ab5c5be0e265479b0ccf2990184ed0731ffa4f290aa112b2b2322818896
+us__scenario_026__child3_medicaid_eligible,4adf06c7f1cb9a99f7ebca495c522ae38c860575ed9f5eb5d7c808ddaa18b5a4,73b56b6bcd4afb524dd24a2086bfdb6e3df9f2e3121be63dbc2958dce1cccc24
+us__scenario_026__federal_income_tax_before_refundable_credits,2b36eae70a9ab4da2716dcdc3a22bb24a343bae2575978ebd672a95b695d0112,3081c5d9442dee3e8b955def7774ad41eb8ad6eca89debe383b2aed5d84d77fe
+us__scenario_026__federal_refundable_credits,0af3536007a3e09e93eff958912cfb987c16b55177f2b49abd6c29d3202c4c41,cc9ff8a3cff796449219c37ad5872cea0df510b378e2b9101b01f7a4d67a868a
+us__scenario_026__free_school_meals_eligible,7c755197149b28e687495c92cef81b8f223082d332c904c632de508f68d22e71,8cde748d4d95ccffd6c193f669506e64d922dd48f907f07629001f555505f678
+us__scenario_026__payroll_tax,f2a9d808a10ff56058a67a72fc2ed217be4273a3b6149a74ec1ca25c447206f4,6cde021a41902a75f515360010f1d740a5ff3b9ff79823423fcc765593eed952
+us__scenario_026__reduced_price_school_meals_eligible,c56c708f436584e21e84cdb3b1043ff74a4184f34c68b9ca0024e6bd52d30a1a,e0a3d4d035ec7a4b5126ac804367ea8ebb66ab08381156cfaf2a9487ea0f593f
+us__scenario_026__state_income_tax_before_refundable_credits,d58084474a2d9e5297ff0735b0c6458b60a95e388802e162194190bb2927b532,9e6fcec5272e99544a79f998933b80bd960b6d680e6332dbf344a9151ae7d5c6
+us__scenario_027__federal_income_tax_before_refundable_credits,9817e0642225ae620461b92156c7076ef9e34471e6cdbc206ea52009af0a8435,db58c0cbad389a3c99a75ddb7c79ef84d846a0b8fe204c91255b722cebebc176
+us__scenario_027__head_medicaid_eligible,2e7c67fa66042ecb5b36c3232312a1c290ede91dfb39945e73b12b56dcee1e5d,e7c94316f511169fdaf10cabc0944af4ee5f5f7458eeff17a6fbf3bd26850065
+us__scenario_027__snap,4b903b6b13741ee1e0177427599aa624cc20f392dea2ecd3c8406bb4de5d8e1a,62b9e0f949906d230b582dc26a226b53bbc005e32dfe1409426b67b8a4c70a4d
+us__scenario_027__spouse_medicaid_eligible,808a92359b1dfd1875c2e1810c3cddf0b6b1e45bba51c8aaddb2eface709ae5c,4d2e892083fda18b2988dadc997ac40ac7f86f69a3a02a34e55e19da49ec1aea
+us__scenario_027__spouse_medicare_eligible,c06019305cb9cd4a3d5ad92e6215a593f85be077abd8f012ce94f53859d7d98c,3e66bee12c86c8451430c567ca95287c46426828099c2135f26abf34687ef54e
+us__scenario_028__child1_chip_eligible,50f8a30784d924b330782356ecf6a545918adfebe7c26368fcfb69a966b2ac24,76fb1d2fbf42022473baa74923030bbed8e998ac468477fbaf502d9c797cdd32
+us__scenario_028__child1_medicaid_eligible,6d46f58fb551089e1e69eadb0554bb373a26cfb45a9d162b313d0ef9d698cf53,46f8fb273c2e5a7121faa7cc6c7400e7bde1242ec6eb4d9cbb9da3589b7d2041
+us__scenario_028__child1_wic_eligible,e2b0b738fdd154b649d1b5980dc348ff4eabfc0d5b109e9daf9d684bf87b5215,329224ec25191668408cfd9575664dea9f80ab4af46b27e880d39ef7cda515f3
+us__scenario_028__child2_chip_eligible,f7f16fe0d177efabc4a6f71014e9eb5215521d974a37ec84c3b3eb0e94fa4931,93a750461570fe30dfe86e834cd07269e6531714fcf57580ed62eec8bd74e70c
+us__scenario_028__child2_medicaid_eligible,fa7bb7fd106751f713da0e507623ea8ee79a20709a013031a2f91ba1cd9015f7,99bae0ce53f2a4fb40095b51ee496bbe2d80c703eb2bdf57ce056c74f497043f
+us__scenario_028__child2_wic_eligible,09a3a7f761bed7a37c567b54ee4ceb4f00feb5bcebf268cae5f4a60d076c0c5a,a9f4f2e7d7bd97e90e850739c911d83596c705a2a0ff275946980389a7d071eb
+us__scenario_028__child3_chip_eligible,9680c066b3949302154b09c36b72eaa76d09cda283b79d6e4c6c48062098edb0,573d972c8d5cc1bc9e181e8a4f8456a3aa54f3e73f9e2bfc5a4afd7c25721c07
+us__scenario_028__child3_medicaid_eligible,9fa59b53f5be06fe828c3d8740721e7b0b4a6a5e85be5e420e12e2ca4d05b3f5,da114a927b181d2d5d2917db3d1c2bdfa3e34e8a97a4515ecf255a9c08a144d2
+us__scenario_028__child3_wic_eligible,420fc84607dad573fd38591ab7217a520c7081b6dcf5ac8055e921a3e464a9c3,1da539fbf580be5f1c443fcf9ec8699c3d7b707251b9632d398f4c9898ee2ae0
+us__scenario_028__federal_income_tax_before_refundable_credits,0e001e99a2ff674540414e5daee51af4af754c9855d8363f658860583d163543,ad8eb09ceee92e824102bd17a00de1d58015c8a045afee38d1cffa9c9709c414
+us__scenario_028__federal_refundable_credits,0faa96272c758e86eef2bfdca37ed126e38463a0a8777c9167f5c13a5b8912b5,5b61c0b57128832d8fcf50309f51cfdfa2e1e87ec7ff1aa36ca4442a6fca6437
+us__scenario_028__payroll_tax,ceadcd70bcef0a9b757bdc0d34b05eb3beb769a35a4a77890e7cf7da14e40bba,60237e8a33a09712b77bd9c9589123064463af32470d3bb59d337c004da13a1b
+us__scenario_028__reduced_price_school_meals_eligible,4c69cabf3d14a66d5904a0bd3ec08f4a4357569559d13943d7be050718cd2a89,f20465304f63c88ce64d3f446732b1eb1007efa991fa3333086e13a42e09038d
+us__scenario_028__state_income_tax_before_refundable_credits,840d599862cabffe0c71c424f7bb0de04b52e01db8204ec853c5d786ec42e236,be39eeca78c5de686dc9306224c77d63523842d622ecdb045cc8dbf180335dab
+us__scenario_028__state_refundable_credits,dedb05f013f4d0bc3b8e4e44e1db641427086d49e71d6f05b61774f995da06bd,8237894159cc249372c961d2d04f64352128db6c99f330232b918c024382b4b5
+us__scenario_029__federal_income_tax_before_refundable_credits,1f08c254ddea02aee96773f0748ec7f6feadd8392023c2969b58da5300136503,d82f764c3c2ea4ac352b0005b98def82fc050ba5ed00b271dffebcfd163c2b9d
+us__scenario_029__head_medicaid_eligible,c0b3872906161eb7715112880328d93a2a5229f11580f3ddb90bbfb6c34a565b,f4dd803e1a9f9ff433c253fb87c9f1133b5842bd611969d5aa1d08156d5ae448
+us__scenario_029__payroll_tax,7d9121c1eceb105fe8b84d4e37f679bf79588c26caf5883e0ad874346bc70104,41fbde72fbf3451e040beb627c841360f5d8162890827f2bc263f7c10a130c3c
+us__scenario_029__snap,094f6578b1a61a88a638c8f79cfc0e728376a2887c0dcb1c765c7949feb89e1e,3b61c72afb3c066cd65360378d23594967b39d799bb8bbefef60fcf3a486e568
+us__scenario_029__state_income_tax_before_refundable_credits,fb9fb74ec04b344c173d0a6d3379ea4c755b98276afec70a306bc2e679d9ef4d,c09d8ffbab567b7ced18ee17d2dd5ede24168af7bd0329ecf1e4610175787d20
+us__scenario_029__state_refundable_credits,f3648d7030c87d14b5abc909ef38ff99c5fcede6801ebcdf9c0b1da9d00f4521,127407175b39b91f2841641216826abb888b420130b0ce6057bc02189ce80df4
+us__scenario_030__federal_income_tax_before_refundable_credits,d1adeadd8022d2178d52801d265c4694e1db01eb310abedcd35ee5ffda8ed14e,c02b8fb860b1d77992d687b09041c0bb3b27f69a6c533a05ca952f055651a11e
+us__scenario_030__federal_refundable_credits,57f61618e950470c1bf3f02f2ef7783d63f3407420223b6e400666257488adcf,8d9890d58f1002307af01c5417c7782021d840e477d0970642ffd776bc66e65c
+us__scenario_030__head_chip_eligible,d16e5feb5fa75542196c3d0bb16efc8812b4c1acfca92e45a0249e2353dcb393,1b466d914250e38efdd7da8d464c353025eace78d75e58440e230b268f7bd781
+us__scenario_030__head_medicaid_eligible,bc56bfa6975337ccf8120ccf0a1c249e8f89a582c239c513d3ca298249aa7c86,1463b28767dd41d43f4891c49be6b7621eafe4d2c07e025d5343987e28a281d7
+us__scenario_030__head_wic_eligible,14cb1029cd2ce2d5aa69ff7505367972ec63e395a1f1890663a8e320bb540d81,90bef848418206359576f3201edc9b39e4ef763fa3334762ece5757d2f6e4c0b
+us__scenario_030__payroll_tax,9c2735bb834a6858cd4b17440a48c9012b04cd50f1b89162a6219a8d2658b086,ad985121c6ae0ea09931a7f8a60593d0e9f1e164e787019973e63e03669bb08f
+us__scenario_030__snap,0159b43cea1f9301d40d0c0954b928df8514c7146a400045b39fdaacb2584141,afd68825646bb9876480bdd2a8b839170fa4ba06edfaa5767109f04fce262526
+us__scenario_030__tanf,fffc09d7657841ad6f1841b8c1720097020d7083b948eb9c8195413a1049f563,44812c88732db5b044421fca8bd9656d514d68381abb2eb0e2e511b6093c0591
+us__scenario_031__federal_income_tax_before_refundable_credits,819c34040c100bb9e43c8fd19dd3dbdfe8743863f39cf0306254c3710782a023,fd09daa78d449c1d07b0e97ebba123b7a30908b070311bda9523c736a272ee9e
+us__scenario_031__head_medicaid_eligible,58a22a3dbe7f4eb77a334b18303ad06f606b56d0c950c9d95012e7d6beddd4a1,1f6c60c1b7ccabc78e9414dfdddb64c36126994b7d6c9c11ab25af6ef62ca8aa
+us__scenario_031__snap,85c386606550649357eaa46008fd65efb08112f951ba98d3ad9e015b4595d103,e0c8e68d739dc227dacc6c3051e6a167438374f2633c05d548d4d2d852c72d3c
+us__scenario_031__state_income_tax_before_refundable_credits,51b74fe0c64842577a891273b3d025e8d84e253e0678dd7f08733a4341bc976d,4d02402567c86e270bc6cbf1cef3fa5e1ea7c8ad77a76c401bd9f252994a91fd
+us__scenario_032__child1_chip_eligible,5c14e66cefef3f6899746e2086bffda086d13977f6323f0f092f5204b24a3704,3b70a6b84dbc7a2a4e4ce03aef678af39664298ed35043130d8b632c58e0d927
+us__scenario_032__child1_head_start_eligible,21cb0f5e9e96dc9d8069a03ff69ec795b77f72c36c0260a6a37d93071be225cd,92734c2fa8c206a054399d07852c1c5865232718994bc18fc43b28e3abd20876
+us__scenario_032__child1_medicaid_eligible,a74733aed6b43492d152961fe2904aeef9fc38af34bb7b8bec5a6b8b7c339915,ac20666aaf2c33d4dcf072ee5fe9c312b032f76158aff8620b09762c07a8cc4d
+us__scenario_032__child1_medicare_eligible,c22019abadd0cb5caf269da903768c4638e5dae7ab960295d6f10390973e46e4,5d3f8a17bccb2dc1081f32f8c9fa073d861706d17f0b8452b6e91ecdc64e06e9
+us__scenario_032__child1_wic_eligible,e3483185e388f5b5bb16af71b189fa562126761ff6305cc4191c5da306ed1190,5a9f0c9a3160a6841b40baca3ff92e059c3df415ca722c52beeed68ea33a1dcd
+us__scenario_032__federal_income_tax_before_refundable_credits,22ec163d64a9f3aaaa77fadcde8592cf7410b2cb8ca0b41667ac11d929d71949,5d0402c111321e64e389cf23362a2c19e26c3d5178cc77d2f9f7da2c10f0c33e
+us__scenario_032__federal_refundable_credits,0960e16795dce7cf2a281c04f1df56efd913e2c6a651052e9fc34cb3d41ff3b9,f17ef9b429e6c7f559eb2d523a7ecb6bc3b8a9eed76c8e1a7c4cf448d51768b3
+us__scenario_032__free_school_meals_eligible,a81379b2b87be0428f63c0d26c12665c1945a1bf0293f0d3c360d647e9bae947,da2079f33390650c9cfef0e9109d47866245c3097e955c286ffb6dcdce3c9c5a
+us__scenario_032__head_medicaid_eligible,6b7f9308589ea2db980b50e3a9854e58c9613eb7a40250ddcee8b5a344a443c8,a0a7690508989a3cf0ea096f9f456cb1da61b4fac2c53bbf1bb1ea08f81ff339
+us__scenario_032__head_wic_eligible,149fb9de84050071b2b2a1d6057e66ef6b4d475c623b4128cadb26111a5ee512,01482406a486e823f19f8bea2bc62200ef8bbf3af2289f1c308fd3f6cd15e795
+us__scenario_032__payroll_tax,73db05d3115b30af9c15893506342899d586d580b98162ebabea032f915abebb,354482e612feabf4d21b5c8d14b96f6d0d4fb34895d8cd83f183e8caaa20a862
+us__scenario_032__reduced_price_school_meals_eligible,b690845477795d811fdeb1ca474367ed8914299e0501a412f25834da6d9af995,70dc16761420eff9defcd16604a73e761ae794da8c3153e61fff8ad4b6ce11c2
+us__scenario_032__snap,5f2ef8a4eb8fe3eee12e1bbe5530b767d51c04e0afc9f030ba962a67f3414e8c,f455c5f34030ed330632516ff5fdc41cfd09d0207c2ee92b971217caa5924590
+us__scenario_032__spouse_chip_eligible,abd4decc8dd5e0255658590c6f58de80d8158845961e2b955797ea69ff077721,531600fd17e2aed79c421c6a66e381983113078c17418dedfcae5ab2f9500e5b
+us__scenario_032__spouse_medicaid_eligible,082628f87e0ffee1293496fbe2bf19a82ed6840bcafa9f0e997f761e243fb817,935ac1049c972dea3bbbcd8569074b935d45419337516f35b9d07edf88abb605
+us__scenario_032__spouse_wic_eligible,4525707f1622925908a5965a705b916bfbc11621159f00d27bb31306ea43ddcd,1fe71fdecdb36728746993cd362e22470b474fa18db8b82c01086d6a136befe0
+us__scenario_032__state_income_tax_before_refundable_credits,989e22da5d9a64d354a1e002e8c0098942b9f698221da16e1d14f30f50411339,05d6027cf5bda1af1f65d5be53f09188c4a14c74658d27b980c46af46d4a5f03
+us__scenario_032__state_refundable_credits,276942ff73cbfe993c9d3075cd70d4589c993ee9cdec5a23df11bcbcefd90834,8ab8ecbf5fa1b9e9bf30e014b791c91aeddf84c69f92c6b45c731afe1b850779
+us__scenario_033__federal_income_tax_before_refundable_credits,fb35e73bbcb63399eca0ddb9808aadda958f828c5d4ca84202d2813dcc133fa2,5c1dbb482df86fbc3ae59ee58962e84a115a0640ddae8cd622fd4f0ccfb066a8
+us__scenario_033__payroll_tax,6b6980b0d668dadf5b1b79326498f7565b533312be8688cdebba90c75b2173d3,9af5039d7abdc4929420e509f2007c3f6061670a7513f8bb2683c129ee686b6a
+us__scenario_033__self_employment_tax,4ef0f946d4cacf90e1ec2ce6e320106fa25505c7888647178b11772ebba4cff0,c3a6d369dc2ee778f0ddaf3347abcda4fd97b6b67bc38f6a389de2dd9459c5ed
+us__scenario_036__federal_income_tax_before_refundable_credits,0ef9ff8529cc26f31bf37237f6d6f23f7ff5c9a7332f098c2dfc09d3832416b3,f22ea1f8ada9191769b8bb449dbb2308257f0f198914552477a669bd0e71863a
+us__scenario_036__head_medicaid_eligible,e39b2db67aee20fed42c4d7c3892adffae0ce958931ccfb352d161e1c12f3129,4725f5b400463224b7bb95e59f46ffa390fd1aef49fc61567b7340663d72c08c
+us__scenario_036__payroll_tax,11cbc8cf6911b85ef615e214170e417f47d2eea81498bbc7d15439de6c8a2c52,1c8845157fc48a774b09a022e8f09cf93403ba5a62e87f9a6c068d3374019243
+us__scenario_036__state_income_tax_before_refundable_credits,44eb047ec49f9f9670ac92d54e9f04773608d47a0bd1b5af8cb6dcaa879f4096,18fcf4ea207be1ae6722c98560c4f540e0ebcf81c43812974530d96ac205ade9
+us__scenario_037__federal_income_tax_before_refundable_credits,eade90c260ddbafa55d637e84401727d5f55495a5bc0d70345054a6174df9d7b,5cf06d77b2cd49155d333591deefa9cb733ea90a29e08801cc593c612573cbc5
+us__scenario_037__federal_refundable_credits,1cdaf4d05fea8cca4227be65912a106921bf9aa19cdc3ccab6636ee1428fd089,3dcd4902d9e873ac3b22fe3cca0cad8c2a5c2c66a9c58e6c641a3c98f2f6a178
+us__scenario_037__head_medicaid_eligible,9dee63f6548a1da2aef8b2dbede00a813c136e6f217230f2e78a871716f94d89,9c93dedda1f19c53a46ba9c8c49aef99255f98165bf87cc6c4088c6f72c1427b
+us__scenario_037__payroll_tax,5e3820d87ef9d8ed2ba56a9c9cffd2fb663daaa43fffaebc549403e226bfa3a8,05df22552cbaec34e23c96c470f33a73ae555ecf8d8b3fe83dde7245f42fc266
+us__scenario_037__state_income_tax_before_refundable_credits,9ec405c76471df6deb04a0cef2abf61cf0de6d5bda680b55eff800ec509f6060,cc707541918bfc2e9ae206fac7f916f76766bc126bf78a5067cd734f26bc93c9
+us__scenario_038__child1_chip_eligible,fb6db1a24c42ecc7eb89bcad872488a45a9b36872b3348dfd193358fa3db8e49,d283bed7f956c0eb83370b0c0ff023cbc0ee01205a1bc3d044ca5e8d8eb4cd3c
+us__scenario_038__child1_head_start_eligible,4680754501a9295699acac70c8e2ba4d19b49e5e6ef9a6037494316e144f238c,94fc0d1d9f2ff635e9828b985e2b42257b9ff1705e4492a960ad77aaa9b13672
+us__scenario_038__child1_medicaid_eligible,a0fb7cab630a01a8802977ec0ca2833b2ac81dff313c3423c23f1dbbc36abc03,e1881b7c2399a917f552c4f68bb915d42d0bd605793ecbd0e0ff15af911ca492
+us__scenario_038__child1_wic_eligible,47019b4b8f2d5906ce077cd3814afabc7eac7ebf6802fa29fa20ea701a427f3c,9dfa7ccb8ec1ed65d40ef4a0f78c3cfd4f8e9eee5c8ea345f072ebac8d0e1055
+us__scenario_038__child2_chip_eligible,a21cab0d51ec6f6e81a896d97df7391166739f2056ee49a44ada793f4affc402,e7efdd3bf3e45ac3594254b8a49b1723a79c651739c7283d0c02ac19740bd860
+us__scenario_038__child2_head_start_eligible,bd9f2415f944741d6d17f7996c266a4b981329af0a03e80f646bdbe3f656ed61,2f20dc7270a97f493343287bdde878e4f4d79599308a62740dd76aca7ea9d48f
+us__scenario_038__child2_medicaid_eligible,5e50d924df8cfacaa48b506b2647eef3cdda67affbac39115f4e218e6462adb7,bac9e1a1215239ac920601f8afe636dbf7f2f9ccd97886b2b8782b66acc80150
+us__scenario_038__child2_wic_eligible,f51f3cc2a3a957055cdff2ffc37e0134c94eebf4ce7c341afcd3b25c509e054c,18eb10faf6d3616c639deec21e8248d6d7ba5b43c8c1d67158e25f5f3f668eff
+us__scenario_038__federal_income_tax_before_refundable_credits,f2898a25fc1d100cde86b30d04c5fd492953e55fcb60eb1a11342917a6e8b7d8,9c27f05cabd6370d201ecd98a538d4140ab5c592a569964b9056b9fbff30d9a0
+us__scenario_038__federal_refundable_credits,fd1b2ef6f0c7f37b840e63ed3e0f9517750a5552d0b567023320a4f37b4f8b19,699d84212cce162ae1d04ab11b60d5e81ffac59fc9fbe86d4156cf006d5fb306
+us__scenario_038__free_school_meals_eligible,e741e90cfb79dc6be87dd576c5479651f3e328374b04e6e1504319b8d59c2871,519f655a6343dab038ddbac5deeae53ced931f7c52b56274aa002e94e9b07b0f
+us__scenario_038__head_medicaid_eligible,a31611a47448d514d62cde0bbfa9917677efc5b93691aa3946ee6a4bf0fd3e08,f3b856158cfdbcde1e44a070d4e776f71400462588500ba209110606296c2c49
+us__scenario_038__head_wic_eligible,b36b1d98223d742f75c7429ef91c7b8bdc0830d30e2b23a475b8348bf7439011,ba0f763475147eec949cac234b0075cd796ffa3de5ba9526fd735fc967b77448
+us__scenario_038__payroll_tax,41a36ad7edafe1305d2f6e400d68c37d02ef575de524585953741c19f825bd1a,964767e8eb52b0ca25e7baa56a9366b087ade120fb6e98fa1611f988a1d7a751
+us__scenario_038__reduced_price_school_meals_eligible,d7c8234536ba690ecb2bbb98c6d43fdd6d98c449e4a3b5e64e82049f5678c92b,b3339fc4f1c4ae9c12e31ae368f1c163395cc574646ccd92f5b449e68d3a577a
+us__scenario_038__self_employment_tax,f20393a7b6f4323d77f1d146de4f1d15abe7b96585bb93e4acb2a2846def7028,27784a4a540b8743d30727453096186eee03fd313c1355771c447239e68d0d5f
+us__scenario_038__snap,a3d7a2992ce4c5bcef60b2b80b9583fb2b74fbe60031e664abeb252f8a98ecb9,d3eb1d8fbbfd5df32733fede72185d8f88ae917851c3ace6868f5ab8d13cf9df
+us__scenario_038__spouse_chip_eligible,6d1c5ca1572b421a39f1c5f2bb5a5b30a1dccbdd125e5324cd98aa18438eca7a,7fecd6364b45019bbb174d6bab9c8d7275648455f69e46f2c9bc471ad0700a8d
+us__scenario_038__spouse_medicaid_eligible,82e888d5a679a410c60fc568c94bcd74652ea758703e4ea56debb6ea2171f67f,aa2789f5f8884ea945895b166aef41802f0cd5315a2e6b624868b3c73b2c2dc9
+us__scenario_038__spouse_wic_eligible,f77125d8b776513c4ae41e8492e5396411ed7ce0107882c107de765dfe1c71b8,d096f6ba749f5266c735fcc1f6d875dcff025e7a6af86a30ffb853ead5009f23
+us__scenario_038__state_income_tax_before_refundable_credits,49a9c2a7f26080a42cbdb384402a5f86ba4c2217499b06d0b150749c56bc736e,a1b13c01246a2e19d4595269d8c667ef1ebeab01fa7856946498835013346e80
+us__scenario_038__state_refundable_credits,0a0c363c9ffe603893231c199f42efe26782fc506d9902c34e7f62dab95469ad,39caba087491166a27fa2eae02e588525ee6b5ffacfe43ec91101d71420f8353
+us__scenario_039__federal_income_tax_before_refundable_credits,4297b3dd110bc534fb92caee458c592f08b1a3481ae1febbf052789184a365f2,cbd779439ea98532f630cd19a27de4153c4467e8edb6e77be62424d99f89c382
+us__scenario_039__head_medicaid_eligible,0ba5e6c2ac7901139bb50233de7b47528377eac1a26b19a36b035ce6dad1c782,a80165745287307440e6bb2b173dc8dfedaa0c99852b2b6418a9d377cc77877a
+us__scenario_039__head_medicare_eligible,8b63512df2fc4fed6256e1d19e014d32dccceca49140467f08b195bef7b0fc27,8bf9565cb90291228576c507c78fa9a60a7ec0b8d502f722ad4125921cd92438
+us__scenario_039__payroll_tax,754dcfd7934899568d3f5a36bcc16455f391f704edbef9221da71be0d6c7f3f7,076258cd4169dcad780515e462c6bf2f95945ee23c587f8937813aaa8e68e09c
+us__scenario_039__state_income_tax_before_refundable_credits,cdbcd91ba66c9a9542bd1054d6fc797d28f9e221afd5a5a1ec985d791008d5f2,d772221a5641374ca43afc675d0c599787d9115e3af3b9ec240515db1c4fcd4b
+us__scenario_040__federal_income_tax_before_refundable_credits,3f672d71c4ee36a64503c28943bbf8241d48698219981a65c99cfbee03b7c5aa,a5ff6471b366f730d62a442214379373964d25390ca21784e79cd82e2234fd12
+us__scenario_040__head_medicaid_eligible,36f71562790ddb5e7fe49883aeaab41245d92b1a023bda582aed544727fbe383,631f4f7cbea809810139f9de12bf5f4c3827963accc309cfaa0bbe9040332acf
+us__scenario_040__payroll_tax,2b5d394a7a6c8b1ffffc3bc46b16099fba6b0009725550275d16b3c069ed21f3,8643dce0ee5dd5d6b3844620cbdca3afe7df39a73d3cf97930eeedcb482a0a26
+us__scenario_040__snap,5475c73155e3f6a912436a1fa2e06479725006da6f96de2b7b7e9c16db95c417,4e5ceb875f7b99bbf1e5222e67793919789490d5953930572f5fd1731554e7c3
+us__scenario_040__spouse_medicaid_eligible,67aacfd380b1d4b23388288f07e1ec3e466eeb0d40585861ed0eadcfe68c4987,b0a5af20fee94df77d32da08b3684b2d02e9b9fe1a99de2e2d0e1a4d8ce31a50
+us__scenario_040__spouse_medicare_eligible,3727bf190e86ecf635349b9bc4a8aa0a40578256aee6f2e4f22365abecbae464,1e6fdb7aff3bd6bee9af83655aaea163be4142ae063fb42bec30fbd283c04cfb
+us__scenario_040__state_income_tax_before_refundable_credits,3aaffb2fa7c401919c52a7b99274443023dd680128f04d71f02ac961792e5960,1fbec781f81396fc6bdc866abe0c1dad865ded5cbf82e37f647988bc421b48d7
+us__scenario_040__state_refundable_credits,4c204814eb3dcdfe0106056fe29502b5cb1e7b1b6f02fbbcc4fe45876eb91ac6,55ad15c86b01c75b543c0e855b174477801ab2922a1f1af1163a9cbab7a1c991
+us__scenario_042__federal_income_tax_before_refundable_credits,a51577e4d07b6eedb77cefb1df2ea6a4047556c8057c0e5347d3e0f557dd0d43,5ef8c3dd18c32b2da2939c0b8adaffc7a39d1dacedfc8727afc819ac2e9f9c5f
+us__scenario_042__payroll_tax,9f75e99227eab776a2637597c8ee73180586c94b24685c374ae42f11f2d85ed6,d830248982203453fd8aaa5af5bce90e8bc968e0f688b7eec37af83dfc211015
+us__scenario_042__state_income_tax_before_refundable_credits,8a2c936bdf4d7850821344d0ec918a18d25f1970528159c30bb5af5bed829550,dd9a07f6d1e65919d351b2baa63496ea48826e3c7d2c51c7afccd26bd014f394
+us__scenario_042__state_refundable_credits,62d14d3fc72f3b01dfdd520e2ee8ef32a6f25d5709585a976b3112cecf45cf6c,ba18c56640dfd549e76139605da2191d1a898b4f206df529506773b004cd548c
+us__scenario_043__federal_refundable_credits,f28af07658f67f40423abac96b08c60a486ea8f6ce0eae0f58760ef6fe97ddb9,d9d1527e1797ba8cd94d148932bdc59352649e17a5521f34d240db5e69a2e93c
+us__scenario_043__head_medicaid_eligible,e1ccc99d8a8cdac22f6bbbdfe1cc4576fcb80ed7c1dffbfd4b1ebfbfebbe073f,7b68064124c7e670231a88f2a2bb80c773c443f4e42c9209b74fc3577f1a2788
+us__scenario_043__payroll_tax,e3ebc8bd7806caf62e96b5642d17387c4404da6f65a0b640ecfabae2a50b2d6a,3baf4b01b10a2d8e12915859767e1f9e366c0f1d59fe6cc7071ce3952431794e
+us__scenario_043__snap,0e59e5c9e0292b7afd79b61d8e67a6e1a5f79ab276196c0bb70561a583fa1b09,1574c02572d427990b211e2e3374fa6d2d2037d33f0645e84b3867a231bafcbb
+us__scenario_043__ssi,3f2fb2e727eb13cb3dd6329b0cc4f1e9d1fb0e6e941a37be95f4dd26ef98015c,6eb8e40d43ed869b65016f02f060870333528d5a4a440d0ec199f405e2438e77
+us__scenario_043__state_refundable_credits,cdb1a300dcd209c9ee88a6bef4cf3ddb37eab4be67195d473f2c647f112691fb,9d5c27489b9e710baa64047e6e5a800b75d990dd31e3afd8371b4cd4019407f4
+us__scenario_044__federal_income_tax_before_refundable_credits,83c4fdee16e08f063240f90069b0c36b1941bc538f3197869cb0ac4010be2a32,1ad9cf7ce57e2e67729195a6061dac993c28b7da063f7c73fdb21219307a0a69
+us__scenario_044__payroll_tax,adeacc729ca9a98af1b6b4d238c66b8197c435ad37cbc852d374f7577359e6d1,519822c2d72101c61ddadaa2e362a77e396c2614a2e7cf4dfc45885b6154343a
+us__scenario_044__spouse_medicare_eligible,327e24a5a80ae7a43a36ab7e0b5d45a0db4b687d96855e3423dbcc032c290c47,6c79e76ee912dc80f51a343cdc636f6637935304584e8747012a52bb5ac6d35e
+us__scenario_044__state_income_tax_before_refundable_credits,4a0b0d92a574ff8b6420904f60cb9cb2b6e3c275459d94b7350837ab2c38f27d,38a30ebdd7cf229e4e9cf7ba2e7e9401ae546ee4fc18ba127c62ebe91f03796c
+us__scenario_044__state_refundable_credits,1d5b7992aaf84ffa48fb15346233f2510c7b300d4feb2bd033e323d2fe331215,153e9e572bd287d680ad073d03707f9eaf8b609bdea363ae0613edaa0d1b7ae4
+us__scenario_045__federal_income_tax_before_refundable_credits,50894b78a469df0dbff7d649c2abfd92d49f3d1ed427fe94ddae59d5e3574d45,f68938cf74800ce3ceb5eba37662cda34d23c217658f4632942b31977af5a5c0
+us__scenario_045__federal_refundable_credits,13546ea920e4a4cf6b9c4968bf1c96122e78675bab5de11cb21b0c8719fc7c18,120817a62c28915719bb23dc4b3dcb6eab05ef11398fc2001b6df00182b46d65
+us__scenario_045__head_medicaid_eligible,6abf1498f4b6157c34f1a080d8bc781c053f3a2eda99a386f129864de26dfe9d,d76318a7a85f55f8ef1fe56dc4ff16276c47675acfe70fbad48cb6066a67114a
+us__scenario_045__payroll_tax,721ad7334fe5f9a80a965997b81e9927953f095ca9fd0fa1ba16455e8b142e84,cf2d7c881b5bd95fe6c7ab897f9bd8eefcf6d5c19f4b51bfc1c66a11688d2b21
+us__scenario_045__snap,b2cbc9c82294274eaf513f55413f8551012d51b914779b579611bd5c8677cfcc,aaa14a511fa37c104fb0a09aa23d8168b5d3bb817264c468eda53fb17d6ef348
+us__scenario_045__state_income_tax_before_refundable_credits,b13fcc727f86bda9f84aa1234efb2bdab45b2de2adbf1884dd2b1a9d4cef9699,c753fe546eaf6cf53b11661a08393329aa9e41de77f20e2180df8b6494d28e7f
+us__scenario_045__state_refundable_credits,eeb1e04043a5e1daf3a146b0d6b8a3c40985c9a2e3dd1149868cd83cff56ce0d,5f218bc5117869d4c83bd16047b6ad15aa9eac7e8269f3d3be28eb119f339ccd
+us__scenario_046__child1_chip_eligible,8ff89b3d57248ea77c5a07f8663ba37dfa3a42d33108f6d723d07991b69d52db,00a659cd188d3cfcfe57724417b48d020a2c501b7db935da822a790fa45ad8b0
+us__scenario_046__child1_medicaid_eligible,6e2400d102ad8a21791349117e82e0015f27da5e5ef9db010e4e4d9791b4cb3c,5e66e6c3456e42b97dbdaee66c0b31cc008f24eb9482aef8bec2d381db24b0e9
+us__scenario_046__child2_chip_eligible,cc7a7faedfff2ff24e739d9f42a0f73729fb23a043bc4253ae3502bf455f4541,a0978af88b17f0d2d5ea14929ea6fd2122fd1cd3f4a1ef2925fee7a5f5d57994
+us__scenario_046__child2_medicaid_eligible,632e3a92212e2e4a7ab0c31ba2fb09457805de753d7f6440508da00ba6d80011,d283b472cf9db20e924b4e9c096904e1f21415439992f817528fc6b59e9c379e
+us__scenario_046__federal_income_tax_before_refundable_credits,db460915c7f17e303056f3851fdabef6a22430fa88f7c20fb3554a9672922320,5509c4beef818a4de894abc090e8cf7b2b6f31f2fdeb73b981b8a773ca4337a6
+us__scenario_046__federal_refundable_credits,522ab6d7784b46476362327048f87ed64145f4c840d9c64616303ce9014b06c7,7b8d903134a9966d7dfdd8803256224030620b63744f2c30b4dd0f4da1e3f490
+us__scenario_046__payroll_tax,979eb484a434b1b1da0222be9101aab630a0ef81849309fcbf5d911e16ee5747,8b54d03ae5a1df19dfb369197ed467a413c577e6f83618a5973d375365e6c9d3
+us__scenario_046__state_income_tax_before_refundable_credits,5377afd0814d93a12b4a85e38e9e9761f3474f7768a2630abb938e4197218035,36fafab6492613ecf0438980d595eb3a8134b963d73ea3e9e0fae758fe436dc7
+us__scenario_048__head_medicaid_eligible,af465bb558082e0762f364dc5ecb6e1078dc2d557900287625d73e148ee36773,d713f6a918118bea7d8db980d5d68170f53e3dbd891a4f4cb74c6e871e20c2e0
+us__scenario_048__head_medicare_eligible,1808e6d7862c7bf5cb34277d5ba9b52806f952d38672a674db04916198f7acca,28260c0de1c264e251b4576866a099d027eece45892cc40e0762bebf17fd4a42
+us__scenario_048__payroll_tax,7dad34f15f6d68e9c48cf260c82b21c1a81ae7810bf5e856d0609e1dde2798e4,3adbd4abc558ae22052586f981c63487a68d2d68f27c738c891d7189e50494ce
+us__scenario_048__snap,1b55c8442a76c3dbbdf04129ce0247c265a73e1d7533a417fb082c4e7208378a,fa56edf7537532b4a8f2feb5b023066633921ad3d831ca39ffd362b9b3e58e8b
+us__scenario_048__state_income_tax_before_refundable_credits,6b35fe6a04e8d1f85c3f85634977b1ad63ba701f23a118459656bf1ed4de99cb,1b7f8f71e11e12cfbcce38e1c085489dce9f98eda8892e85dc3188f2fa13b581
+us__scenario_049__federal_income_tax_before_refundable_credits,6cb7971270c84244e6dc7e20fb719d955d77a337e244422f4bd2c51c53599df4,c8053637803c354196b59c198f9c7465ff5e2faac5c91abba0973ea0efb31d9a
+us__scenario_049__head_medicare_eligible,508a5aade6a873bc35ba9fd69f2459eb4de25e5014864cd1599e87b2131e08db,14574d1382c0d1236a4a507153300b4241958d91c0807e6ed6bea507cc5d1a82
+us__scenario_049__payroll_tax,91440d005dc2223bbfc3df9d0b4c7601791a83163d0503b3b1a6c57d8395fbf9,9c86329ca6d4b2787f19018dd1b0a2fde8e58ed8110b8af5a1785cec6de88c59
+us__scenario_049__spouse_medicare_eligible,5f2683eb144a866907725559634fc32bdef2dce3f8edc03b49e30a3442390fd8,a0b3db99aebc9aa0a46a192d8a414b15f6e134bf77efec7995267ba8e940afb7
+us__scenario_049__state_income_tax_before_refundable_credits,d0b334d42c5224a0836a87e42e8c74009fba48689e78bedbbb9de1aa532cead3,57c95a2b4ee4823d60e17f31d5c45b71c2f2ed2ca2d82f3f4de6124998f587a8
+us__scenario_051__federal_income_tax_before_refundable_credits,85e0e1f46086461574de76e350467695734d1900e7d58798bbd8efd173de3c27,6ebe19ce21aed104cd20e90091385ee819eaa08d287ba590ca3e38cb42ae4ddc
+us__scenario_051__federal_refundable_credits,6e6bc4e61c06b828850b4d0dfa5489c47c5486bdd1496c350a9c1ca6eb430116,ad7a510e71955355b19f8a00ccbe064d642c767b2cb5853f638c2f1108562e4c
+us__scenario_051__head_medicaid_eligible,0586fb51dce2e1ed38e98e3c028de7d7bb759db9dee6c082f0d4d7a62b438b2d,4256be67fb88ced26e6fe68b169c1e1a553f3ec35e90dc5ec5e020c17cea7354
+us__scenario_051__payroll_tax,8b3810f6ea11cd7dd4221809c58cbf4b2cc3c42305eb315380a3995f285ea209,52c20d04d1d7cbf60ee335ad28822e6fe7c5e9d1fd1b7cc33af2205c6da5729b
+us__scenario_051__state_income_tax_before_refundable_credits,ab85f8c09287fcbb5c6704610371403232882bac8e183b9f42204d548816c603,715e6d6c6407b9c70d44623843ddad47965f0b09ac762fef4b38625dadb32aaf
+us__scenario_052__federal_income_tax_before_refundable_credits,439d7b875a09ca597c3d0f552918defeb0597cb94b973d0910e7e080832959d6,0d50886c3b0a7472bc9f8a061a3d3baa59a41e0b8db829e05f1bed30b05b7eb4
+us__scenario_052__head_medicare_eligible,164e17b4745fa0ef4e25039a0ef8068732ae3a546dcec246c84a61cfdcb805a2,af58ec2c96a3a746b0ad4c6eb8e1eea22489c2a579459b5bdedc9d9f0d33c51a
+us__scenario_052__payroll_tax,a0eecad0a09852a8686aa09dcd226cf0cf0bfdd0fee23dbcb635d1c0c2e60fd8,77fac0893496daa2e734842817331f11700291b562d4ea0f565da265f6a61a74
+us__scenario_052__spouse_medicare_eligible,c12f7c907b26eac5511570a2cc0a6cd9b2f0f932296d848ee2cf68d4ace562ab,f3ddd1bcf7f2506f75121b24b4cd69805bedf69457e3c44e019602e8cdea1a3a
+us__scenario_053__federal_income_tax_before_refundable_credits,383a20693200c05b252edc0e75d3b07d90f7f71ffc619b9969cb97e6802a3d8a,9db6f4b18cc6877553027a58287e7d59bf18c2831f0e1ac4517210d0f54fd2c8
+us__scenario_053__federal_refundable_credits,ffede74685464280f003d47044a5f15147bd05f69c22e68ec99884f9d0f50c1f,15b94adadeaba3a4b146f4bc23a477f06389d1ff11e121d55988f5bc68c58b07
+us__scenario_053__payroll_tax,fc7bd38198ce59065233a244527aac93996100d5086cfabbc0fbe3da3be52a30,84918d4565f6b4220ad0826a53567cc64bd17f0d5a230e0076502ddcd8763a60
+us__scenario_053__state_income_tax_before_refundable_credits,74b0aeda0cdf50542788aa3df554b9c242599c8563860a2727b5e290eec0e2a5,0c2fb9d8cdde6e224354a0a291a8eea4d90c7d336cd588037f4a6bac5ce746af
+us__scenario_053__state_refundable_credits,90982adec7967aee24fbfde8b6c1ca0442c191a8ed13064490831e82638ddbe9,2724b39a99a342c4f29198e14e37169d322a1fb39e60dde9980ad38bb1fa63e6
+us__scenario_054__child1_chip_eligible,e910c6e99a7c2b82e93f415f33fa07f1b72493756e0d177a06d11fe1d0711709,99f36d276b92f49d6638f422500ed312ca85a1bd09334cbb7f6ca5c860a29725
+us__scenario_054__child1_medicaid_eligible,755ce7ce7b429ebecb898e41db45889a81e3fb532a488ae436543a438760ca64,780d9c63ac2430244647c0a3cf8b0e2e139686b2bd49b642032d3be324242914
+us__scenario_054__child1_wic_eligible,ebf420a222a27cf59b7d6378d04dd6e26f220d9d6ddcccd70d769784c844d633,59c6a581add091a9d9c395526ccf381352213cad6bbcfea5627009b0261c2a5b
+us__scenario_054__child2_chip_eligible,4e56930e77963c77c11b1dcde0db4f7c0aa6b1a720b0c4e35df3c327a737549c,cd86aad7f8dd4e17624e9a99f337ab94b2e989fad77b6e326391522b8cfe87df
+us__scenario_054__child2_early_head_start_eligible,1b6908a66c8df12a48e990792d1f70c867072608b6af72f925d40708412a61aa,50bb5636990a7bb880353c2079577a1ad0289d911fffbc0c92b8d978e284fa8d
+us__scenario_054__child2_head_start_eligible,e54f05a192cad6896ef57e563eb120fe3ef6bf407dcab7add1ce96689bcfc85f,7b7c98f5e43441e6ca18573bf030a073ac6ed5fdb517d82c19f3af50295e0492
+us__scenario_054__child2_medicaid_eligible,7915be52b82cf6a0feb8293209271869ad1dec7e718a990c8080a1e336e9b420,ce9abe300df96a44969ae510aabd67fc18cf2bea6d1aef25e64c50655563e9d4
+us__scenario_054__child2_wic_eligible,73fa3d6dae841552259f48b922a155d40d7797027efa5aec2048a95a64e63bb0,2a20c57db837283d7745cac1c1f876366f51cbb179728799ffe66ab8ffed0caf
+us__scenario_054__federal_refundable_credits,0698dda2cd618c7bd1665c5f8fba5f39cfe565e52498d5a4ca5b8d641e3c2e3d,beb3643182158791f5818ef6da20370b4ded61ea670ed96fd3fe162c727b4610
+us__scenario_054__free_school_meals_eligible,272b41f318402b8f8281fc7015c8f20af1afdfa47bae9f705720de7fe161304e,0fb737c11bdb437e593ac92ede33b104338a0ef2b37b50e47f07e5466823a570
+us__scenario_054__head_medicaid_eligible,17e5abc53ec3bbc718a912a326e94a39589d7e7dfaf5355521d8bf4fc07a51c6,55f985ddff08fc45b85169f79248661951ef21fe907254e639dd16ce8936adf8
+us__scenario_054__head_wic_eligible,25f02c541066aa31f6a2c2037bdadab41d7899d90f3855bbf52084b7b844013c,0ce5f6996b56c8e716934976320d228185dc7e19e367495186f072dca8858e9a
+us__scenario_054__payroll_tax,456d183aff785f6755222c483364079db7049ba9738400347935948cb4f6891f,564378153453428d1a8a02c2ba3dbfcb340f1fd1f99a201b9d6e8b2d6d26142b
+us__scenario_054__reduced_price_school_meals_eligible,f12a56a4660c97ba4d7f5df29798df060b2955d9ecb829a32e6e5811d890e88f,b27fc7c3d4106e3bc6c14f3d348dad0cd19c0b7814774a1a0e0c1798bc92833d
+us__scenario_054__self_employment_tax,a8acdb6ccb643a3820e3b3ca22a76b301690f22cf7f556c70010957061b182f7,34584872334d8dd119b96788585fca3af74b4e8e9cdf2795d187dd6059c877f6
+us__scenario_054__snap,2500a2fb20a83e0666dd3cd6cb9e791e07dbf4e7ce1c387412acf3061de2b8e8,f9af944a45870ba5749f0a34410d74d771a958a727ee67fa4c73a8193e9c74dd
+us__scenario_054__state_income_tax_before_refundable_credits,a3e88e18ad91a4baf3c47c93e7b6222ccecc8e62c5e2b3da93f46935d1b83b1f,3baade3fa32e68dc8facf405f96aaeb7335fdecc08574f2fac53b3de85571fe0
+us__scenario_055__federal_income_tax_before_refundable_credits,f18892c3259f7d032f26e3e0253fcd5525adde7110b71f839ad23889d187c29f,cf0a53f92114cf43b729b1dd2a81edb23f65ef69fde6cc3eec2bc53372f1aa3a
+us__scenario_055__payroll_tax,d679db023a368822952b62e560f1d87d125577422fe1cd55dd09aa809cd09f46,3e3d36d2e8c525f216592da5e54c991e39f7014827b0c3e1e564fcb01207793d
+us__scenario_056__federal_income_tax_before_refundable_credits,869fed4267fd38e002f36fa8c77664485d37446ce7f17227f88b651d7e6077a3,ba7abd54e65ac9d1269f5a042a2f76e33045f3746893f3f3164135cd2e0780f5
+us__scenario_056__head_medicaid_eligible,8cea184c97d14c48ab06ae60786e2e218504e4532b3d05dc5daaf77617ac7f0a,8804b0dc69774740c7590ef027a716e1d0dbda44601e402b6a7cb4ec618b8993
+us__scenario_056__payroll_tax,6531a2cf3210005ac2ae383bdcae048f99d03047c5e6de4736637171d9a3dd00,5cbddaf39f01de2b5d6a0cc5d7eae0b40221abafc0709b317e06282cf8bf6a91
+us__scenario_056__self_employment_tax,7901956ba0a795ba0c00a0c60afdc4e588328f7098b18e4cb3f85a81d327e81d,d2ed557c1f4ce1a2beead21aef264a49b77c71f1b7a5e2d8c98546a890fe7727
+us__scenario_056__snap,336a19e45b5b0cc2a9658f56da6c5e6ebfd91c4b611e372b12b0c89d2cac338e,9f2946cf1c5f742b98ce344ade9f5019b5dd1028071132f2899e43cf95ae98be
+us__scenario_056__state_income_tax_before_refundable_credits,828b862f6768e81a9585f9ae508db46ebd09bc4c7eb4bb9dadcafd573f2647f0,2be4296e7aa80df26a2403455fb0ec1d3a9ae7fbf8c87715f4cef8fc89ab7466
+us__scenario_056__state_refundable_credits,0a94b3e11e11dba191502287da894e3fe2fab7ab908a30b10deab10f82585905,8fc8100a6f962be910fae4d4e2cb0c3427c8175c403b54179865d92e38f2665b
+us__scenario_056__tanf,e29d915d8be36465f330cd19283eeb444e984e459bb631b80b28ede890585dae,c9259662a30b84fbc876aff7e14b7cdecd76b47016a909caeaf9f45390d70b5c
+us__scenario_057__federal_refundable_credits,3f9283c0d7c688fd650e5de4e59758b9e74fb8ba86b0d3e3ac2414381c65662f,857cc27db2a6ed2681e8352479a993caa1a94b503134f7ac33988e1776467d2c
+us__scenario_057__head_medicaid_eligible,232324e443d96dc96c2f845225aba9036508fd7fe90471703643878a3a619c6f,919a4a1120ec0f87f90ac23e44903af2adf9b48201830dc2982060891ee68d62
+us__scenario_057__head_medicare_eligible,31640356d7793022ccf13f6639322e4ff8e96dd40caa05901f69fa24c3222eb3,2c9c1fafca0415c6c08052268e58cef0eedf2c7cd7d846d4fc48807102231f83
+us__scenario_057__self_employment_tax,5ce219cef2a55c537084fda62318edad4d9ceebd1b2907131db70215cf6e2687,024e7c4c5998e7c761d8a235dba4191a7ded0b45843ce21ae92e9cf14d14fa4d
+us__scenario_057__snap,cece0091ebe154f8d85418525dbc25f1359d9412723c15d62d0bf34e30c61fde,3ce5834bee87f370256da271126c8a39ebcfa0ac97bb9354f483f3c25193b7e9
+us__scenario_057__spouse_chip_eligible,40ece4779468b0368b2a671ee61fb8fcd292b92b38b0771797127becac9b3190,d24757f59ecaae276e455383add3bb6de3af7cc5d25ac40f3bba805006598ff7
+us__scenario_057__spouse_medicaid_eligible,c9d80fe640b7c9227e9c5eae6586229bb8c65c2f5ddb7bc41dbd4032fda1ea96,d1835cf06bf4d537ab87d2cf05b494e2fe3ec69cd7f16dd80bad05c1eea7a219
+us__scenario_057__spouse_medicare_eligible,9594fa0ae8fef7633cd0f0a5e8d487dec07647568ee2f8fe045c42f3db365437,3f4931cb8fc81f78b74eabba5762d66fa9e2afd0eed48c20b254ae9ed8a0ebbf
+us__scenario_057__spouse_wic_eligible,cff4ad62140e6c2b165cc678d22144e5df113f38535884b399283ec3ce395179,5a5d2c00c386a9da7842d62f48676ef7c5912bbc094c7c689ac6784da04b60e3
+us__scenario_057__ssi,c37ba2d66db96161a362d19ead4580ef801569e15c53eda6919f7028d63465fa,e2bc31014415ca8925f93cb858288424e40bfdf6847965d0f5a6c4a3794807ee
+us__scenario_057__state_income_tax_before_refundable_credits,aa152b05672b14cddbbedfd10c01edc052e23b7948665f0d2fb02b6e66c345a3,6155014a283e4a283cb844becf06bdc5e67668f64d129b48d24cb8106ff54376
+us__scenario_057__state_refundable_credits,8553f329d63d966d787afc1164a2197a49ed584d159325e811c620b85d28f4dd,b945291b4e0b66373f8e9ffe70a3c99455059b626669c47d721dabc8b1c7f8e3
+us__scenario_059__federal_income_tax_before_refundable_credits,1d439d6a4477451fcba34f7427a23275a3de177fbb434c8bb2169ada2af69e4a,9dc6032d8cad01227baf3b1f015650b7d0357e076fa6d7e40794694e92a9140b
+us__scenario_059__payroll_tax,419775e1c08e04d6bf1f077f1982d6845dc0f391ee5a7309abad54ceb7d88d07,7693f0de9b470f8ceb671723561f7c1f75062244cb024eb40be97ab1601dfca9
+us__scenario_060__federal_refundable_credits,ab74a66d3dfd2ec829f5e47e906bbc0387fac68fc4c2940cd1bef51d60c079c2,f2c90191bd630870458cefc9d4cb7ed54f90af5e95152059365a924ef85b9a09
+us__scenario_060__head_medicaid_eligible,afe47b5a9a0f5298686c27bad084339c0cbaafdebb1b0103723605e8ec1c34f8,f9757bef2887b783351fb7ca4737cf76aacb2f4d638d70f60943d01c09dcec0f
+us__scenario_060__payroll_tax,99b393efce61001a1b7ef18e1934ba1f186a93770060701b9932e454cef62ff3,e2900a1d724e071400b61eca7563b1635d6a2e532542fcf75e9dad0b52f51f1a
+us__scenario_060__snap,72fae2f9e5738454676e15a0b6f23b9d5778ea7fb2dd254f1ffecbaa371925af,494e0586a67324fea0722cf480096c813695e55f29aa949df670e8a5d1d1b4e8
+us__scenario_062__federal_income_tax_before_refundable_credits,91cf3ed639be434bffe3e1be25380e1881d468eec906d512f27c12fff34214ec,919ad24833f5031ac9f4638a2ce5067f2740d429c683394f833c517ca2c5c4f1
+us__scenario_062__head_medicaid_eligible,174f8369cc2f5c91b7ceef7d3a98275d257d9c56be745e0a39843c7c46732b84,97a6ce548ebd03cf6c3424c54c57971602590f8731110058b7f1ceac80be1f8f
+us__scenario_062__self_employment_tax,f9890892974cf662e6002cca7eedc3c2b616f76346ee872214beb93f686b071a,8f686ccfcc253f315944cfd2ce35c0f7972dea79a025ba6837f7f05defd695df
+us__scenario_062__ssi,e1462635938189f55b8c9d75d8a5219a5631fb0cfaa7b49f6759e777ec44cf1f,b7b62aa2f60ca77d8300fff19b817d86faf1192700b6945bb9551d44b2e6a5ea
+us__scenario_064__child1_chip_eligible,5d365c4e2795b1793b190832385354135429d96c17f03000668bc6e648d6acc1,0c8b99b0d8e1c42d4ce88430902504b12cb0b0e51d35bb6888a891ab55821bcc
+us__scenario_064__child1_medicaid_eligible,a163306ffa832c6c306c97077ab18cf3edccda61d7921ff1de274ef695f64570,6ee633cf42cc5af8031cf2044791e2d91795893c5aad3536c619bcc121f5c8d9
+us__scenario_064__child1_wic_eligible,1f471bc60507ac15134edfdf89f0b723fa478de4d363a66baae1d77af86a8bd0,40c1117e868d81937384ac0ac878ad690fa6e03d9ed17d0aadfb1ef2737e7b82
+us__scenario_064__dependent1_medicaid_eligible,1bd4f46e6c2a106618311db8e984d510c65f906925fb4b6e90efcacd4ae2ddb7,8173e4b5fd7befcff8b654e1e66a1560b53474b93424a19e3b3542350d92acb9
+us__scenario_064__dependent1_medicare_eligible,64c4756c41aaf49f4ca5af5aa8b8ebc05fe73a1c73ed4d1ce6094f14dcdf8259,79b260b9c9748248bdfb551055ad8497b4fc7cb1d97da9bc50d31193b56e010d
+us__scenario_064__dependent2_chip_eligible,bc53b95a138fa2c27903cd3b5a3507c9d4a3c97eee1c17172fa5c27c31d01d06,868fc9a16564705f61ad6a51476a5c558fd92ba01e7e3697038d1eb5dbb50575
+us__scenario_064__dependent2_medicaid_eligible,c482b5c61b5624c64f9fc25b2049ed0b99feb1a31917e2b5ee8e4ff894e80a47,e933bc244c77b2293a1138231cdfb903b45b479b17b43995681953fcd89ec44f
+us__scenario_064__federal_income_tax_before_refundable_credits,40dce5f8b40e7c3b880192fc9da46140e40d8bbb5711c17ddd9e489d4d8795ae,682f09574e3a06d77672db1f024ab0f850ecbd32a8f55768da84f12de1ab5a51
+us__scenario_064__federal_refundable_credits,198803cbb4e457ff11805fc6fb99ed0efc12cacafacdb48b9d7d5f0a6c2372a9,25387f1dcb0642d4ac5e11ecdb8af787451c7d12734afc3630988376e4bcced4
+us__scenario_064__free_school_meals_eligible,e8e503f32843f5a4cd62293655c0eae89d7ac85538d77b9ce9b944efc647985a,c4257c9d0d823e5d54dd3988a6cf87a6bafe76ca4091c94375cea5ef418a0c10
+us__scenario_064__payroll_tax,9664b997329e18ca93b1ba3af26f86fec41b844c7dc8d16075ba48280aca76e9,fb482bae6bf94222c33c0ff46c1d89a64d7ee131c3b535a730ea71c5a1640d64
+us__scenario_064__self_employment_tax,eec323b1b8ed51e273b9ef58b1a0c588494a76e0d177d944f27402970eb18515,2d0bc850240854ac857104807b4e3d8175335aba56da72ba8dad041af6d2696d
+us__scenario_064__ssi,7446e2aa6190aaa1c04a025f644d524acb3e398ea5d5068a82b5d33ed3633efd,947269714de4553457a34b370a8eefec212a129a1c6404e483d9db21e9bff78a
+us__scenario_064__state_income_tax_before_refundable_credits,1a0a330a5edae5c2016796cd1c39a64cc44e5111fb4f991aa720aac8b699356c,4d23f3ff02192426d17ba1608fe5b069d17d701a3224ad345589a74d7a332f58
+us__scenario_064__state_refundable_credits,2d31533eb9728b969695198c8016eb43d7226cbf317011fd5caaee4922e84664,c7dc836f5ddbe8a8ec015472b276a4d8b1c9e4dc1908f8ad1f16a8a62ef66b89
+us__scenario_066__federal_refundable_credits,e364143a6ece2c379e1697803158e39a5850ee4a5c7dd167469df5aa80e2771f,c166cde90b9104849e3db334e1b225dde77d694c7c6c4ff3b13b4fb0d95754dc
+us__scenario_066__head_medicaid_eligible,124d6c83eb99fc5208213dd28aedefb00dedbbd488a2c40b42929e468b79bc08,47b8aea10e6dcf42767d89b767b766704c4b4e17526a6af1bcfaab32e285221b
+us__scenario_066__snap,8209611ff60c7e4de6ed8a12dbc30f95470b0cce960cc5b36939b845de30f268,0b3f12a3795ea34ac276c9245317ee7337787caeb44b664c08c40edcd49ee827
+us__scenario_066__state_refundable_credits,632b4145c87de99d13c55f1d244bd1a059209a8ddd6a62c2f150baa59c9ba009,7cc7d739abfb98e2e195d3208a5aa91b35ce71e7debd3211e2385558d70e6a47
+us__scenario_067__dependent1_medicaid_eligible,9cbc76d33e89a9d62f135f2b95e8613f4f9d085c4bd2aba9fac6ee7f8c317dab,fb827611e784dfba133a4174709eae2f660a3dde77563313e9821e2d574097be
+us__scenario_067__dependent1_medicare_eligible,6c82d350debd4d9e97a14bff9422dc0c2324f7fbdbd6512bc6c54988080c97ee,1c65d7b163e8a3d03d75a45ff8f15dfe10e2a2a7c55ea9c0fbe9816123bb787d
+us__scenario_067__federal_income_tax_before_refundable_credits,f0a03d3dd9dbbbbcf6f06228895b7d1bc566df396749241cb9340fd5c20a14a3,14b3c650086550d7d336129a91d9a09a761db827c6ec633e3c846f81bc1e248c
+us__scenario_067__federal_refundable_credits,d029f05fad8543b059707f9b3a0b2eb2673f9ffc122d33c166eb5c02d89fa187,8a04a288d5a4c5fc9367a8f6abb9bccc8f388402e23ff7177c6ac8e55bd20ee4
+us__scenario_067__head_medicare_eligible,b52a10fe439c3a0d9fea4523fa7e5d5f5eba94fef9878311a24196a2b36c480e,e420b9cabece63aa44e206a9f4148ec01522c8a821a49654efbbb8d30c62bcbe
+us__scenario_067__payroll_tax,4ef064e3a112773d5fc87abe5dd119a7b656c53b06d65f1f2d00473d8427ac6e,7167f712406b9172e2c02c5df3cc7dd4a6700aa24696050df175698622a13898
+us__scenario_067__ssi,0659da5aef8036e3e1b9b1ffe87b8b89e96bdab43ece0fdda2eab1e7adae123b,d8ead7f6d91041aca7013a1313ce3350fe4afada33ee2d5d6fcf9b08ac8e24d4
+us__scenario_067__state_income_tax_before_refundable_credits,f4a4705adc49d7054404172d79f66be704eb931f7e7985742f9c401dbd40cb58,dbc043fd81f0818df89db234c343b41259ad40307d957943a777a905ac7744bb
+us__scenario_068__federal_income_tax_before_refundable_credits,e5997ffb253c642c9a28efaed746084fb888031409fbe66e36faf12f2bd34082,8eafb0abb87a757cdac37994fa7bf0e6f7c4d72e90f328064331934f84482573
+us__scenario_068__federal_refundable_credits,6300187f606a96161e337ed0da3623b89e0040ed068c4898352bc2061d8d93ce,ea4b061ed07ac0f610687dbe9767a5d9e626851ab1757771493c60a7e358de4f
+us__scenario_068__head_medicaid_eligible,19f5a72cdf5d984fc842871eb8607c4b549466018bb36b6f04b7d6be921e3e16,8f59d588f88ebaf855be9b388ad5aaa5e9b2b3e52b202227033bf540ebe0ecae
+us__scenario_068__payroll_tax,a43ca39d6de4cb7807845586da8bd87f1a96badcfc83fc6bae9dded21622cac3,765d5220e322360d9d68a2b808cb2d098a013cb6b7db45c43123b9e007fb84bb
+us__scenario_068__state_income_tax_before_refundable_credits,31e9a96d7be18b5f6a5c5b061e25aa03f4414b5d3002c4b689b0abd72f59f2a1,b98a8b36a76a11b130792409457a67122358a16810199127e35e668c56db2034
+us__scenario_068__state_refundable_credits,4a19c8a306292486a5a2ffb5ffa7d23db6eaf519425ba6ebe18a427bf7d76adf,a90a721260c18f62ea2de2c672d2320ab7aed3bba2a75e3a74e53f15e394abed
+us__scenario_070__federal_income_tax_before_refundable_credits,17a232f0cf32d934e6de2898a97172c8747e736c8ae7d6f8eb3c5404a1b068b9,8206795d51cdf0d5ecd424f695ebe24f95a9ef148e137af6f61b4c9531189023
+us__scenario_070__federal_refundable_credits,6c984702dc99967234e1448f7c37a55d07dacf5cba470a6ddb36c9020b4350ad,c15a83a6353cba54edff881cf585fc3752a941320ad36c3ac3b387cb95f0119b
+us__scenario_070__head_medicaid_eligible,3cb246086d3f2633f7559ded9b7adae70eb773fcd67223e846bb08181a8f068d,42998370c8e6bc126140806ba78054929b2828c1fe2f92e7e41bdca17c300c74
+us__scenario_070__payroll_tax,d3830d6de8663afb49d463f3951e804b12685f2abeb76d570f9443622d758f59,1a49a2cc9cc655655f1cd3f580552f4939080350e7b000214a43ce7654506c3b
+us__scenario_070__state_income_tax_before_refundable_credits,d5b22ab156b4f4154fd44abc4be81d5d1bf0612fef4a4103324b85ff550c1c86,6fa410c1064cd206b438b9513472c83a3b1d2abcd30b93cf97005638b145f5ee
+us__scenario_070__state_refundable_credits,d6a8f5765042bf78f21616b0c4b87a51827ae0a769a0b39c7448507e2a35608c,678dd72b80d6ec456f5fd815d0e0ead8894860be0022e5a087d5122a45330d94
+us__scenario_071__head_medicaid_eligible,ba99ebca759d39acd30547f4bb2f06d2161374720399b22be919a2983c7a928a,dc1212f95319023a03f94eae3467b623a7495aaf08bdd995ac6ec7d359391cea
+us__scenario_071__payroll_tax,123f3d610b3d0ae82b3f4df3a9f5bfb960671a64c47f6bbe6396c7f3202adec5,6616462501479e935116425369020023f1a3e8c39d4d6fd3f53cf00064ff2551
+us__scenario_071__state_refundable_credits,6840e2782fdd59bd46ce7d5aac8b2bcbf2d550da8bc1e91078ea2b4a85d65b60,9ba83cfa83c3e12acc3018f44d1d6ccec02368a9981a68b3afb7b3d1d9dc4b54
+us__scenario_072__head_medicare_eligible,8303ac810b4482adb25fbe1fdfbe1d0154f91d4d3cd660af81dc728b5a352818,619695951d21cfb3b3ce679f6d4057976f9277eb3ceff6fb811f83b7df8b8929
+us__scenario_072__payroll_tax,42e1e96ebaa613af49f2faf9686b75e3164179d2571f3aa394bb03455c8384a7,c61b259094aa5a9a9380f1a0bcd27f06227930baf8f1ad4f19a9ff74387248f1
+us__scenario_072__snap,99d0d803c8e9c2a40c60c14a032da91da14afda5941c513890221b41135c3847,28cab303bf314c742a14f4dcaa304f582f5d2d8cb88e4f5e34be97ec0cd35d48
+us__scenario_072__spouse_medicare_eligible,00c27b71fe5cb873698511e6cf4d986e5e6b3e25d067b5b633628f441adc6cd1,31386bfac7c095691dbcb3ec771403a326a679b4ff0908eb0803760e4e85c31e
+us__scenario_072__state_income_tax_before_refundable_credits,4eb78404c03e372ba9c32dbd8c41b4e306a1d07a65050e5bba6f56fd6b1823ae,0c694480a2a62dfd8aea7f709e1feaa39a5593bcf261b9d316f85f66effbcf4c
+us__scenario_073__head_medicaid_eligible,0ad62652d3553a704ea065d65b8fc242d0a255be297af0d390c6c87beb56fd64,48965a2fe7636d7972f4aa0061cefc5d3304f2c4e667e4d1155636292ef8a70b
+us__scenario_073__head_medicare_eligible,0a1fb04f38e096a055cb7c73ba1455602ba851cd6c3c7db370118253f8167cd5,9e871421c786a853c005cbc79fe0284def24692131427b59955eea5516a6b8c4
+us__scenario_073__payroll_tax,62a8774e28a1d2403c47602a40f50751c068f3063a0df96772719ad687c9b7b2,dfd6807014296d6db200eef2d57f1e2a6ec3e63eb936c6b28f8e1d6e649793a5
+us__scenario_073__snap,36c5d358ecb8cbcfa832562a5700a6c0cf3ba1b9ea75ead99c990a8dc0bce5c2,c465fe7cd2b1ac93b46f53412d80373dcd4bad2efca7d061a86ece2dec8061cd
+us__scenario_074__head_medicaid_eligible,84befb4c8867cc4b47c5aa511c00b1e3cf709262c4ccc6c93b66d26d0f8371c4,f358889317dbe6c6c2d77f868aa2033578f7592fefcc40cb64537bcdb567394b
+us__scenario_074__head_medicare_eligible,c6c14f58861e7c66368bf32460134eb74908c6db6f8fbcef2753bdc64313bc84,f68e05f6a97001dbe04f873c9686524831c455ea6783211e6ceea307db622f92
+us__scenario_074__payroll_tax,470de108439fb9d94ca606a71cfbfa8ebe2e807907f172934333e6c93cd4ecdd,3ee8e6be5541322047c977c78a792ac03938004e431df721d89d37526fdb2441
+us__scenario_075__federal_income_tax_before_refundable_credits,b92c9f195f318666a6a49cdb60657e4500a4e5e21b804b073b366f9ac3e39c9e,caed73db9ab9e1e99898b927b95a4ed5aef6735121063944db18383901f863eb
+us__scenario_075__head_medicaid_eligible,3301d5f151624e892a7d200692c2f9b3286e339f68a0f52b4a14ac573d770521,9285a211798d2f119a35c8179164fb0db08cafe2d72ca3f7d195eaca3f367ed0
+us__scenario_075__head_medicare_eligible,b772cbd89b76a4e8145082b542fcb3869161d4df299801bc9782283c9459fbd2,d091d4ced680f4c21d01f489eb8fea087e509897a6da4f3018ec802c457620ec
+us__scenario_075__payroll_tax,0a4cf42598637457633767cd80a96f894015be79362b187fd6537752f7b387a7,ebf4dcc8e62100166bd922d47f650942942a2799d5d1ec3d5e278e5d659d9278
+us__scenario_076__child2_wic_eligible,dcb324e790cbf06a106c3cdb595c777872c2eca48a3a4ba1f57a0b5348db4ca0,d47eed169beac46c7c0f8ad0537e54e5233515739c6aa11849ec639935e0f93b
+us__scenario_076__federal_income_tax_before_refundable_credits,69549e322bb9d4db30f8fd590200d12fa5267b763ea32138a21e016c541a345e,9531eecc0f39155791219b7123a8700e55f4364e2c570b25783f85d9e6304a8e
+us__scenario_076__federal_refundable_credits,4125a480c2c2b3f05ef21d1b4b30065f70997a7b9419fc144dbc332500f21fa1,1057142da7c9a28cef711e0e5e206c0188da6869e7c415f99eb7868c0c0958d9
+us__scenario_076__payroll_tax,cf731d2ccb49b1614772ace7fc7a47afcb1fbbd9df341c5e0f06cf1ef31d47e9,2a309f139ac532c2147fe024d1485aeb3f3181f2253a489e275bf4ca086018f2
+us__scenario_076__self_employment_tax,2d150397ae7fe5ef0a663ea7529317a421fab3c9d43adfece5b2f9dcb71a9fb4,56cb0733e2e10bebf17bcc2e51ca2ab7dd0b5aee26eecc95fdf5f74a0d625228
+us__scenario_076__state_income_tax_before_refundable_credits,520bd8a94466d819b93cec4a5202294d74065920a55205ba00606d8474d70c7c,269a36cd25c325732cb3a4ca4c10cec03e3bfbfcee23fe9dd9af874c506ad7be
+us__scenario_076__state_refundable_credits,37e166f66e3e244114112f07c12cbae121ad76c94cce00be2515e177d40dfae6,b7fc957b5576a27f903baa39180eab7c7d5ea6ddd532f6a06b267dbde86ab693
+us__scenario_077__federal_income_tax_before_refundable_credits,191fbd441f79c23d157d93512e7183710809a78d1ff8be06b37673e9a2cf3dcd,0507dba0069f170a79a4e2e5e6202ffd8986c7ef46b18eb86966b9e3c19c6442
+us__scenario_077__federal_refundable_credits,b8f45460d30c69ec4f3b5c14ff1acdc762b6d82fe9f553621d345bb5c5859153,be84ef35a55e2793d5e8a790ce803caa6e270c26003936463169e76f0c1a29f3
+us__scenario_077__head_medicaid_eligible,d72d46c9a98b4b56dc463aaa6553382adc2442bb3a184d3d91222a38c5749899,7daef34f5bd68e1748ad3a0ed17772bce6c440b4ea37f93ba8897de2d10d7e8b
+us__scenario_077__payroll_tax,338a29146830eee07f27635e2e63eb6c6a9785b752b595b4cf6d402cb616868f,4d836d7a89aa1405b0da7a4b7e03057d1d94c70f8189e1727b1c84d5a5975967
+us__scenario_077__snap,42cd315c4574a478c9f5ad080d0c1020c3900f0706793796c24be4bd13691110,a9c625950a6df86d4d71c926ed932aa92f4a86f3d272012e9242f57ce95998d5
+us__scenario_077__state_income_tax_before_refundable_credits,82b4cac7df74d71917b79419fc995566722e1367091e3d8580f3813fe6062463,b92854c4962b5a229f835f737ea26446381a2370c7e2072971fa6ec567d1938d
+us__scenario_077__state_refundable_credits,016fd89e08a42ec4a55ad0006c9bc9999c5bd0038ba6f84a328a9b40a47db34a,eb290063dd4670a7b7879f0dd59d665a52599fc1669426a7c830194bea20558f
+us__scenario_078__federal_income_tax_before_refundable_credits,027d17c5393e04e9b7572ec0b2a2ce6b323c0c0231183d201e7ed660b3e2803d,86ec921ee90c4541a3614da6cedee28084de18b5df277653cebe96d0ae91b7ba
+us__scenario_078__local_income_tax,ae75e1f800c367a941799f18bfa5df88046bad46d93395e67dabb75c8020f187,6ec4769f167ee3cfcfda8e2b42a07aad5103527f4940990caf8849b3cebd50d6
+us__scenario_078__payroll_tax,5e2ad2cc8a3a504e76ce068fa40edb19f4b9f2bd56180658e1edc351b6521845,d288b3157c5b4b9baf116871390ad03692a24b157bc225bb3d48fa9c8e69b29a
+us__scenario_078__state_income_tax_before_refundable_credits,d18b08c450c3215502c5c1c7f82aa490837fbea7b274a5c48548267b7cc2f780,5834d0cd99c13e06a872d67afdde3c429781faf384ddd206ae15dd58b212c90e
+us__scenario_079__head_medicaid_eligible,54607808aec56f1a6d052748f5f44a5f63ebc6ba602de245a30258d5f9375819,d13733174e7a5e03ca4af5124d5a084a74817c97abf9aecb24fcd9c83b2da0a9
+us__scenario_079__head_medicare_eligible,eb612da71ac1d4c6e92ad2e300b37508511f890db7fce9d273fd60bb5039694b,48dea412783c086b1c96d15783f2da65ebcbc3d3f7d70df722d829b8ab7e7201
+us__scenario_079__snap,c58b8b26c6834515b9363d10bdd197b857950766624607072549a83cc6bb27ae,76215775a1915ba9bded091f443631ee8ea3157d52e3deb3c60652c025c13f94
+us__scenario_079__spouse_medicaid_eligible,c380546d28efeda3bdc439bdf5eb9697e0f1f6329ddd31e18a908af6af7e862e,36397b770011d07056985f6ca8492d56f84bdcbbf024c37a9fd78151576edd92
+us__scenario_079__spouse_medicare_eligible,c57f8a3099906ba8ac283017b9a470c2c522d126a621c500daf75936be7d4c07,2e385f74d441774fba06673f0649355e114dca399983ec8c6ffb4533a438702d
+us__scenario_079__ssi,1c9c70b5645425d697448c47895f553728c7d6a6ac21fd151e7b9a90e10a3dea,630216cdc7fffaa68090bd1b70e9aa6791f63c3f93e5bb4fc16da615446e3048
+us__scenario_079__state_refundable_credits,22e6eeedeb11dd9faecca7542d30c4600cfc7bf0d4ad5a1db1dd3c6dd0f5df62,3fb107075977b8e14e76897ddcd76cf0920dfeb2a4dad674249e7cacfc039c6b
+us__scenario_080__head_chip_eligible,b9e243b6e9c037068d659a827d4efaf40abd5d121f00858acc06f269b5ae1f5f,2bebc812bdcd3afb3421d9cfcd80f85c031a54519c059a9ea5b62a5bd832d008
+us__scenario_080__head_medicaid_eligible,16d1b05f6af3005da239eeb250450ddf94b48ec2bcbfd41f3619bfe35a634db1,fd63fe651f2a41a394659cb102c39e1df15f02821352fb72391743d4f9c031b7
+us__scenario_080__head_medicare_eligible,492bfbaefcfc3cf4d7f171d47776f7c047bc20c284c26e9d66b6300db4a7ab44,42129dfd841242f5cc66cf5d734ce1a8b079879a1bf9d95136305c06f3308fc3
+us__scenario_080__snap,6d04d061d9494f396e801b0553e2428f9fa6307b8ea52b6cdb2b01a2a84408c3,45d0e5474cf178353ea7eb3e651bd83a439010d74a6c697563cc95c49ceb80e1
+us__scenario_080__ssi,43ec21461fff18629819bf6759eb8e9c588bd39df5f8995dfce3ebcdc481c8f2,84d00efc18d7c7b1daa09fe96695cb46edb33843d7af0e65f0d3d7d97b80940f
+us__scenario_080__state_income_tax_before_refundable_credits,26ea6c8424e9afaac05bf877993a8cf76a402aaf6992178465395a8abf77d26b,65e8795b8cbd17ed44252f37465d7cdb50c6ecf219a48c336691cdb1fd8c706c
+us__scenario_080__state_refundable_credits,95beccf4253f75c7438dbf00dfc205c887dc689e56002fee1d0b4c18105a4972,ff2fb3e2da9cb07e82d437f6d9a4ca29ddd983170e1876ad3a7ebc3d68e86b9d
+us__scenario_081__federal_income_tax_before_refundable_credits,b3572286e87613afc4f07f82cfebaa8c56b0d75003beab0af1c8040e9a3e2fe0,6b046acbaf2b36276da1ea1f7fdb5b3cabe075ae69e156df9594f7c63f44cdb7
+us__scenario_081__payroll_tax,1ed3c1e942399f545efac0f9785aa4bd67bf22498ce01336a1c4eaa89bd9d819,b29fd367d87b47db08504f5116ad0dfa315fb8b570074126b2689b4bef9fc54b
+us__scenario_081__state_income_tax_before_refundable_credits,5c559666340a49f8fe770bf1ea6a7b0086edd6969d0e9cc5621c6ac0236d9d38,a82fcb8c5a610207f9b8cb83f44dc710d1fbd66110680bde9376360aa129387c
+us__scenario_082__child1_chip_eligible,80b33e55fbf3e28c423c7aaba899db4f361fd63fe4174366cf3a0af84b2559b5,731f2b677eaf50a70fe8968a3ac828a4fa48185dd863e74bb6c5a87316f0b99b
+us__scenario_082__child1_early_head_start_eligible,d241170df6fc5ec2029d90f9fc36e3c0efe8f52334fac0783f87e6d89f63f7f9,6f50e20abc79a7c20c8e238114f31f4ad62d176cb1dbad5bf67ab443dba826e3
+us__scenario_082__child1_medicaid_eligible,d4c90f759a1e2b96dc2246cc21623d0d55b418986ae37fbafb34d2f65d2ef1f5,f2f7943ec6b3367fe7205f7f9eefd68110bc23da84128c82960ec570b09de8ce
+us__scenario_082__child1_wic_eligible,3be2990225770847807aa5a221c2e4b8aae867b804e2e3eb93b39d925badd0f0,165c8776674e8d5807c4aa4537e26810fef06866a5ad4578193e9103628271a2
+us__scenario_082__federal_income_tax_before_refundable_credits,4ccc8cf90b753bd19b0e943dd1a7714495e11ae9227308e67391f1582eb77e85,8eb2255b38c4bea8860787ac0fcc0737f0056e521b9c6ada4e3fd1679590b272
+us__scenario_082__federal_refundable_credits,c14b0ec60660e88c83f681ac79565b9e931def8d84dcea0590a0c99e47740a03,da5db67eeced7d31f30ad4a4775297e571fac1755b5118b03629a4c3b4d143c7
+us__scenario_082__local_income_tax,c5b87f8b4749b02085941e27d09aee571394a5de297b9a126eab8dbb3a5e3696,e7790e074408c946cbace6724e8bdfe01086d00180dd1199765c7918b9b798e4
+us__scenario_082__payroll_tax,51557f6b625f9f70bebc0ec9af38223ebfef4704181afea8235669074a842606,b7a8964530c183bb40079292b65272d9387a758efdd583b53e239096e06858fb
+us__scenario_082__state_income_tax_before_refundable_credits,cfb5b6bd3dcfc5389b5bf51d61e08f8bcd65c72ff28a6d7c0f2a1043bd7a1fc0,a7c16a8b6527e720aaa55759d09b0b8486f5dc6d06d84d438c352203b8ef11f3
+us__scenario_082__state_refundable_credits,80ff2aa18f348cffcd657ca5760eee9de02f55440adafe8ff3d8022244de5503,55e8fc1cc9a2db2b5042ec2b0b663f30ae7a037c7438f6cb41a34f115e414fa4
+us__scenario_083__federal_income_tax_before_refundable_credits,ebfe45d55e2ac1aa9edd1ce1c1311b2972a11be334240e20605b367d4e5091b6,73e0da8cdd71ae02011f35c379191fd015f3652f82048893e7afa40e3a00a078
+us__scenario_083__federal_refundable_credits,608f65e5e547b0b5ec853ea1d43d57eaa0d895ed98f84d8ea6cf9b72a1e887f6,117ed52ffaad02a72621d18e1c940a874549d1cab93f06eb676a74db9adc648e
+us__scenario_083__head_medicaid_eligible,8358e5331be3f07cb279f791570254a115496649a01a60a36c1c6df4f46be714,1e04ead6d9649b46349582885ceb3c20cd5f4592c4fa4a238d9919a7cbf2f64d
+us__scenario_083__head_wic_eligible,4435607c931ecb6c4360844bc887a7bc38e9dc9afa7f19a74a94457d3914352e,9c87962fd37c542e9a6272d0dae6388a94115e560ba1bb116bd347e05a97844f
+us__scenario_083__payroll_tax,fa01e9d20210c5424321ca7882e05c25bc0fe860101e43d5ca179fd19236b77e,c80c2cfdd4be89b14502a5c9a739b9308bf5eeff37df32135f5770aa926bf052
+us__scenario_083__self_employment_tax,7135171a2c6cac98702a69c4b2942f96153cd6c2123cd81e8e6bf3f958f3a085,7dc7d9549fa10e089409843468037006efc72d517049285383f48a1bbbcb74ec
+us__scenario_083__snap,847f77b72b10efe0c5b2f6273e5130dbd0e727511e215cdc07419ab55319acf6,cb1223419824e574d2938c3247d031169ff0e1902a6483b496d0975ddca9a23f
+us__scenario_084__head_medicaid_eligible,0e5e90dc483e8be74b2d68446065a96b9c585838afcec38845f7c8bb57d65534,88e6a367b44a1e25fc365bec18aae59c25f3b077635fdb2212ff131be3f4b284
+us__scenario_084__head_medicare_eligible,0c7acf46270a753352b70c1dc435a62a4be5ecc6b1f0617f5adcd3a9f5129287,1afeb86061c1a9855bdc8e6cedb8ca5e9c6d837e7aee86ae8284cdf2ee37a5e2
+us__scenario_084__snap,31df3f430c2ff4676f627b356e2ffa8ea2a50cdd37d5f593e9daf1fcc401545a,cbe27bd9c1a4f5aea880cf52058afdf402cac8536dcdfb776d6c752ea881e73c
+us__scenario_085__federal_income_tax_before_refundable_credits,e444cdf8dfd4276c27691ed083fc9c2f849aa678be1af0c1f506c0b0647452aa,2feca4723fdab959a645df2ea2720b71e1b2705879cca813b0fdf28e7c67f37e
+us__scenario_085__payroll_tax,d3d31aa47604969bc9eb8a80b68fd9482a43146ff55ecfc0c187b1f33f582406,10b9eaad094fe3eb3b1402f485db0de6c5fee485ca11738722c4ad76decd18f5
+us__scenario_085__state_income_tax_before_refundable_credits,2e0f74a999fbf5fb8f13cf8a7690164e2a41f0be7ed3650993edd8e3d0aaff22,c3df674a0af1d45e867c59650e0d7d8182206f1b316eabff57df900cdd1655e7
+us__scenario_086__child1_chip_eligible,132b2353232480d388f6b4fe9b18719217ce2ee58fca95c1bab94eb72f38081a,8b42d7488bda25472edc168c6cc71eb0ee1b7dd3616892e5033eaed742a301df
+us__scenario_086__child1_medicaid_eligible,b93c423803d89654ef4205ecbb9450f810b164b7f0e2a494159edc8b3f4f25a5,852e7130b43982e057e00b4f4a0a49d178faff72c066725543f7e6ba5ceb86e7
+us__scenario_086__federal_income_tax_before_refundable_credits,9ff321da9b79e7ac831e84f6f1010325f8db33634035306b85e960a7507538a6,1c97c59a94e665898894b9350df34122ce8a9fdee7e32b4610da97aaf42a14f1
+us__scenario_086__federal_refundable_credits,278696e8621e02462272b88b837c08e4a6dd4ec2518ad57360573d0fd9435d1e,101ad0a457764903753416f9aefa0295e8fb6de8a583407675a3e7e20f504de3
+us__scenario_086__payroll_tax,daa030ce5a2659241a349cc5010205281928f2367e846c278432ab7a30c1a2ff,5a4c6d0128dd3f73b6f231f93181ad95820781102c12c9e76da2ab171dce6b90
+us__scenario_086__state_income_tax_before_refundable_credits,f88dd8b482f9545fbd469106db10b5a4f76a6a9dffc6183d3d9fcb85c18c58ca,cec83e770def7b775ce87be715428c1cdea9b52fe41ad724f3b012d72e633035
+us__scenario_086__state_refundable_credits,ea1145fdbb6d74270c3ae2ea376d9aa3a345c1c12c125bdf8b139a6e06b0da0c,40da83d0513aa388507d9c5a60bfaac4e222f1fc2b8d970984683217b43dcc7f
+us__scenario_088__federal_income_tax_before_refundable_credits,e1bdf03a8438ad2b82d2d5dffdb81acec8c63e18ea0927ad3710a23ca68bb1d8,80265259c57739dbba4b287e55eb8edc58985af7d438b4dfec98641d640d9039
+us__scenario_088__federal_refundable_credits,1b4b87155061a5567d6f438eb8c9507034b9ba0eca8b8e7386a2abb45ac5f8e6,548a6ac1712f13b8de12e2f882aef60f22c960556f3106fd4da39821f47ab74f
+us__scenario_088__head_medicaid_eligible,f262f7ff9be7bddb5a00cdee6dbe87c0704c5956d0b93b5b6cd97b8400109992,c94f5e5cac556c5a4826900a6b3793a7939a53ce6fe38819f5b9d8deefc45bb2
+us__scenario_088__head_medicare_eligible,21ae3c74efb67aeeb20856ab39ece2e8545fd6802a9a4db7d8cf2e37f5da9099,7d444b630e33f0cca14a5d33e062b05801ccf72f83ae3b8c015299b54e311d77
+us__scenario_088__payroll_tax,ee728e53172e6c8d1a81d8fe48b4d77018ff88f0d75ebee0921134a06b3276ee,c1636310cee9bbeadd30d5d11afa023f948a25a9330c35f9d86b4cd470e41f5c
+us__scenario_088__snap,b513ef27aa643ae954feb25faecaba19236c48352e713e0a023956d1115c0f8c,c2749c18b4c65d93ba5e259c14d9d1ebb9f1d32fa02606585f5407983de1c37c
+us__scenario_089__federal_income_tax_before_refundable_credits,3d68cae1e41bad08afc8ac7a818f8d33cfcd68f193ed67d64154b220670ffc7d,4e54f6c83fd85a9bd710918616eadf12224858cf88f405e7d12784c23b5b070b
+us__scenario_089__federal_refundable_credits,76b8ffbf111908e61b6585203ae5d2b6ad3a88576729bb525d54a1dc3da1e0f8,abffbc6622c3590de80b2b669cc6b0cd2bb82bc6b5ebfcf9d4db194102637e4c
+us__scenario_089__payroll_tax,96842922b67d3e023cf25d54b5dfe48e3bac8da826b839370a74a7d0f8633488,b4e53f11d2a23c11f94689168db3628f6f96706e436b5fe72c7477da6723d049
+us__scenario_089__self_employment_tax,af4f031ef8c78d06d632aaeb63644df33d0e815f5abe75208bc1f678698b3fd9,6b3a3325e09e82d44ef5f19d476e92276b8f27a0b8bcebdbcf7756b1f4f0a377
+us__scenario_089__state_income_tax_before_refundable_credits,3a18ad686b3cdfac6fe768b12d6c6216451c2c8ca078cfd962e2ee7824c4d487,e4a22e22d0f8e7dba38df25dfdb9739520789b288ef944f3714b17d5fe93aa0b
+us__scenario_090__federal_refundable_credits,883b9d8dbff5f97cf405381d19b5aa6857773f235705f7ed3b6e633181928a9d,0b986c5eb2a684246d25fa0f7b066d3f6664a4b46d150c4cce4899a4a13c3705
+us__scenario_090__head_medicaid_eligible,975518354410f94fda216542642828d9172e924dc5dd5312a6292e316ce5cc95,b2bf8589c249f839f0d978d9f85d4d37bf37455c466db33bcf5547446b7d52c1
+us__scenario_090__head_wic_eligible,de7963217884179690525723fd5e09cabefd68b1fed7287ade0df00331ed0912,cf121c9a8d4a8cad7f923bcb84e57a761878545910ab9745e14f92cdc6f9a9e5
+us__scenario_090__payroll_tax,0241e654bff01cfc8123680d878f92de0b5c8045919cc6c7d34d7972598b7a28,ec28e561136033f45a11d8f085a74170568bb6c6bd2a1b8d525eb152a308972f
+us__scenario_090__snap,8b4c8cd1fbf6e67e564afb530189a2db3a9ffa4a064ff14ac5f513cb0190daee,e2ee14fe34e800822456143b8d32fa78b83adbe276ad36e517527d0fe4abf4ec
+us__scenario_090__state_income_tax_before_refundable_credits,9bf19f37bb7c98744890933cf4026b4d9ce5c0b25597dff3b06dc154f5761920,2080cd651c58a1a1cc934a4b1a1c0bc864d7648f4b787ffae81f9d0c759a9e61
+us__scenario_090__state_refundable_credits,c69dcf0865eb11ac1db95892523febda5700e6f4cf89c009ebdd24facf4c27cf,b7830c44d0e70c649f4120439b44056b0b8aa160cb07335687e64e09754491f8
+us__scenario_090__tanf,0d5de46899b23929efe04fdc55ccaefaa9ac6fed2a89f738f3cf646a16fc1654,4497551dc0871de67850eb45ec3df301fb19656905b16327354b507d2418f302
+us__scenario_091__federal_income_tax_before_refundable_credits,c8013a1179a4cd1b06c41e5e3ec8d6eff244c9f34a763501cf4d865504e040f3,f49caa51aee27d372c9d7d1680b911ac2cefaad32961876d32bd8663510f0a73
+us__scenario_091__federal_refundable_credits,8df1f4ea06a8e5e061a9360214c3eb0d2aa9cc87b58724a5166427fce19f53c4,b93387945c5ced8c23931f580276818d481bc9c6fe3ff01595fe3a1e41976a0b
+us__scenario_091__head_medicaid_eligible,09a593f97e8e0f55da018a2c9dc3bf5d71862ebadc6a1f10d828feac711e3733,bc1816dbce66b03d332116782e6b3b849819fc83821bd7f0e41c1a1afaeabc6d
+us__scenario_091__payroll_tax,264502af818f1fef1c23b60840d544b0d784f247b7893139e0a5db39ac7efab9,14b1e5791e33c8a7a168e5ec27dd8eb1cb6c52ec2661a8170cfe5f14a65a4dd7
+us__scenario_091__state_income_tax_before_refundable_credits,05c6fd8b19d81199582a58d36e922ac5471c88ab2bb9f2e69398b43d322ccb7e,4d9fd3e14368e2a435a9e767cb358384fc714643a718194645dfa8b15762327d
+us__scenario_092__federal_income_tax_before_refundable_credits,9eb3008ac41ee85eb6a6e7109170d6762b486b8f3b1a0eab767ad1df5004fc21,a6a8d17eda6ba980a1db0841174d0f345b686bb5b24c454c5655dddccd776189
+us__scenario_092__head_medicaid_eligible,d0656f0e15f47f9043415bc39d9b5b1eef75d2cb375a0463806980e068e0192c,ee9ba0e26fa0b67c5c7796ad5a338e1ec2b0fb7d8409ce1b14bd4cb41d28df08
+us__scenario_092__payroll_tax,05ed96e35072eada4a402d80d6455b04caea25572462171004960e696c717e2b,e53420483a6035291619ac59fa234908e0101dc75891ff043783c9f8ea2ffe9b
+us__scenario_092__state_income_tax_before_refundable_credits,a7150d425e823dc20533bead348bf37380d391e8fa22d3137d70132cd18c359f,838df2ec9b676e9d0ae5359830a848e99f91fd70245f608f0f9d7523b4d04617
+us__scenario_093__dependent1_medicaid_eligible,eb1c9057f0e3cb56fdf807100bb1a683550fbbdb421c0f5f236da96c28302a3c,f99246c546c6d8c8d52f3ae8c0f9072ce6f1f3dcbcfbecd58f1094abf9a61b2d
+us__scenario_093__dependent1_medicare_eligible,4c98f5d5e065d92ff76f3323c7b589f75b3e80876e179c3544ee96764806944d,64ec5368210534eb7ad9108a5fc76fa26d75feafefcbf0c1042c5c17264934b9
+us__scenario_093__federal_income_tax_before_refundable_credits,bc3c5d38046448b394deeb35250bd58d02e65a46ba25bb30e05af722ecda5f69,792814c5eb17b3da4cec7ec4e82053c4d9ec1073fb1a814b140f16291cfd3751
+us__scenario_093__payroll_tax,39be37b0e8114e372e82fa73047de8e9f0dcad8d78bea28d5a3d9c9653704df2,d2bd8568cbd17219a008634f12ffba7c7d79d883da16b2456143199f0968a7a3
+us__scenario_093__state_income_tax_before_refundable_credits,1570f713f4aa199ef0d82eb713683f6c1155b0f95d554b63166d93a2dcf19407,f28e688b40b6611d5f47152422c2391b77b0b12123a4a34a0ce7d51c3dfbdb17
+us__scenario_095__federal_income_tax_before_refundable_credits,ccb5c9665099f1d79a8912ce862a72df12f8fb00805a0e82f82f719b93303b14,91791c2297010efe95154511008a2e3d4dda5025fabad17253f3d8297d93d551
+us__scenario_095__federal_refundable_credits,b8bccb7e5a85fd9b6635acf822bbab9961528b69d361dc678efd92e392d2f2e7,c07bd65803460c28987589426954d25cf63a87e43c627082b58df374b144fec2
+us__scenario_095__head_medicaid_eligible,6cd67e0f6cb5665b551e0b9b5c3f83076d9acae10dd5de188266ed30298a516a,257446f7cfffdd8ae8897c54584ea6b445118e09798fd32e98a0cba5dd350e5b
+us__scenario_095__payroll_tax,1c474b2f24426553c37f93431a143ee7c8c107d0fe1476f2ae5f690da317c3ce,ced8b7c0a5b863ce0ca9aafb5590cee7a4a677a2848f806dde38ea6d1ad5ceb8
+us__scenario_095__self_employment_tax,20143b1cec485e2263a4bb9acf4a43db0c6519de11bee4d125188e136af5d62c,f57d9dcc03cfdca601a222450c567e38c14377ff8761d59e34c36ebbf2650cc3
+us__scenario_095__snap,098ae23c19657b353b7a0989d70d3ea33b4c9f3721ac59087b78263f336416af,aa7021baa80f939bf546fd2535dd0ae16620d3aab855360654c5db84764581bb
+us__scenario_095__state_income_tax_before_refundable_credits,a87ade1b4e8e9ee4bc49463bad701d16e0fc1b8771d00b034e2876123016cf7f,4c892d51b8c3cc174ae824a90a11422666543dad45514cf32c14d40acc1476ba
+us__scenario_095__state_refundable_credits,8558bf722f09756d755dc89f429284b0363eeb7a824e4de5a377f89a4fcef70e,23f674e32065e89a52cb52c82dc54bf8572cb2e40beb971381867c777ab66142
+us__scenario_098__federal_income_tax_before_refundable_credits,a38f4b835a8615fee9e27cfc0ca40aef6c0147b04557ffcc0c4ff32c64eaa212,ec0835e4644076760e0c9e99a307ca4f8278766322d505d908502bf615fd5b97
+us__scenario_098__head_medicaid_eligible,8cd262afd7a4a5b417aefbef9b8e5e0faa2c6cbe9421dbd6c4e682bf64320080,5dc5b7fe103930e61b86379bb10b504b34a78b5c76b8d16a0dd27a520c54ffe2
+us__scenario_098__snap,b70c95dc088b00f8350eb378731243909cdac4379b57e884ff1cbbc6a565e262,d07af42cc921dd613def0a4b91761a0f0e00f600f021c6896ffd9ee03f75d6a2
+us__scenario_098__state_income_tax_before_refundable_credits,3cbd357144aec83f90dc1093c44612e75515b8e3e4501df85ea99fb576480e04,1ee09ff45f612781bd76ae55b794d277525bae38f990b10edf80c9b6633ca8d0
+us__scenario_099__child2_chip_eligible,b0f31b2f454547fe5d09dc6e639c8c4e63989fb675bf88bda01f713c545a3c52,2ea73dcd67efb98b2e7d6f0f267882c81aa878e35b41a9bdc9b7a6bb04358715
+us__scenario_099__child2_early_head_start_eligible,2fa95a87c099ff8ac433c5b2b8d7624ee6144ea3d47a7590ae159eb9a79e77b1,7b908adc23d068598ecbe1c7ef1229e4109ffcfd64159586f34c4890a30acfb8
+us__scenario_099__child2_head_start_eligible,86da19ad621cf5e66d38598fe8e04c7d92e12a7d18fd096c15dbd08065d4dbdc,ceb659b95e5d3b715ef5b5a5a454e87ea1ec5201fb99d0a1078c40d4e33054b4
+us__scenario_099__child2_wic_eligible,fcbed40dc9ba79bd8a1424465919adcd4dc22bb89308647182b676f05ffcbc10,3f3394172da33877788a16cbb8deaec82da485bee529b664be1e9c2780deab4b
+us__scenario_099__federal_income_tax_before_refundable_credits,526fb90150219b1c37f07fbce5d8b88ea30d8b116d61128e10fb025cff7dbedd,1d383125725f09931153816a158373de61fc439c9020b14a896cd472a8d48633
+us__scenario_099__federal_refundable_credits,168e1138247a6005842f1b2525b8411c148b6527f5c4f41f68a4c82918aeb4fc,de41a2f2039a46d03bf52dbfa99e68a28039d42c1a273e93e5ba5f4da587195d
+us__scenario_099__free_school_meals_eligible,f004dfcfee69aeec9e7bcfcae2bb3660594df2c890f59fbaeb115a4b347fb675,bb1e171903416c3e880b35d9745e0a0d3c4bd6191caf6985a75fc7e4e9f2e3c8
+us__scenario_099__payroll_tax,3a7f05528b9a4b49ad84e8839c8096f979316aacfc06f0e774542ba0250e589d,5d5964c8ed0a3873aa02f72a0282d61e301fec582e3fc10d24f4dc8673969e4d
+us__scenario_099__state_income_tax_before_refundable_credits,d91ddc4f604f213ce12f792e806c1bd6be0e1c8f741598742d5d24ac120e7c3d,92a6d2bb227353c4a55e3a7e170a3530dd337ccfc02ad5564b3e68a3830ac7fa
+us__scenario_100__child1_chip_eligible,7ffdbc7ca5d08f46ca46ef66182bc29a8c46cd19a72a87925e2eaa4c36d40514,92bb560355412c7e0ebc905e8a42b418aad216365b267c7272ce94e55ac5f87e
+us__scenario_100__child1_head_start_eligible,3a45c04a38aa6d578d3f9bcc7372a85cee7651e5498a49bf590595fc4ce95ed7,e461745e43a775ad6610aa7b32d3f9744820f64ac1c7c608fe6293779755a99e
+us__scenario_100__child1_medicaid_eligible,f4d72f7e9ec7a2a3c872bcd432b325779ba1e11df4929c9e5c21c7b880ed190c,1d1a4add8b5311c807c8f9ff80c58ae1dd96477788525eedd426296fd953f21a
+us__scenario_100__child1_wic_eligible,b93165ac4af96aacdac7ab4614ddbe807465a52a9cffe523de29dd09f7e6cc24,7dc92815ef7b7b2f828b9e4083f18a1a2bbf182180e6ab46847f6c909bcdd828
+us__scenario_100__child2_chip_eligible,fde0a7af991c6e5e3740c523584c06a4a43ffeaed5c9b6febd2bad6d9a910429,b09ad5050f3a921a53a2e3185ff5a6d41efcadbb366b85b3b026ccba41247a7a
+us__scenario_100__child2_early_head_start_eligible,c293f7bd6ec6d82d1698e3fea3cbc42d8e812b9003cedaa382769b50631a4924,37412541482c0bb137e2db632405a91b0b55bc15b4f2d40b00dcde3d3b285d72
+us__scenario_100__child2_head_start_eligible,6fec1a2f819664628af449323dabf4d3164be30ac606a39c9ef58fa9436970a3,d12046b335644c4a08ae44c001a1759f6910d7f63ef2ace53e15ff26574840c6
+us__scenario_100__child2_medicaid_eligible,fe030897de6850c05bf4ebedf681a49e2c56fa9d5935b651d2702155c36599be,231902793dbcb27611ef0d54fbe06ded22d5c10211053898d714c0dfcd97a12f
+us__scenario_100__child2_wic_eligible,15aac302048c0071ef24dea934b0db74ad55fb9d1f0d15a37db249bcb57a3efc,dbd041e195efadfce756c8f63904babf943637aa01ec27b88b307b5a9486f990
+us__scenario_100__federal_refundable_credits,459976995dcb2ace4864948cce8f6977b3abbb170fca88e4fee8d8d66ea9d1a1,0eb7d54533c420226f4e566a5929fe8d3343e2337b6250ef2a43f0816ef34314
+us__scenario_100__free_school_meals_eligible,7228854ed21c25c78f6154b660ba69b627b06c3a83ee5f214927e83d16d897ab,346b3cfc914a2538ead9b42441edb562c301cf7f59d04eb601c3825a759fb31c
+us__scenario_100__head_medicaid_eligible,8923cf8df4c4e6e63caa569ac41c858a1d8a7100eaeeef75443d45a05bc3bf4f,96081aa4f900774be344f8f0354ba1040eee81fb73a457623caec09828f39819
+us__scenario_100__head_medicare_eligible,ad947d335e7a019c438dd97f875e7b9e58b25aa08455f97573282a67e2dce0ee,0ae21bb28543c90f5963805ea426fe4027fa5b581dbabbc47360e3592591e49d
+us__scenario_100__head_wic_eligible,cf153d90fd04487b2e320b91ddd524f4f43a7d4486f8e9d5b7d4288f2cc16b3a,a419bd2dbaa28c22cfbf65bb7e9660ee4fca6d7157e5c153f24f71de0416eaab
+us__scenario_100__payroll_tax,f864eaf0d9dfe4da779598c661c52b7491350ec4a6528e4730514d81bacadf61,2ffede13456c8f1cbd057431c0494a09c92dc28c6391cc1c6d40e249bd2b6a90
+us__scenario_100__reduced_price_school_meals_eligible,f48e0f5e3e1be80d313f5e14147df9a8518ebbade9afefaccccf8a085743b7e2,52a1034f6edfb84b8ec945fbaf13dfbbca8de2b289716d76054ae88a909c9364
+us__scenario_100__snap,c0581bf082b97c424029f114efd9402558395b300865a62f6a3131eb522d35b4,5e8ab5b7201c1cde5aea494bf7de631b5c3adc9f51a2f05edefa8928a8bc432a
+us__scenario_100__ssi,09ed7e493c8ac49b0075113658c5d540283544489f4c4a4cc90936015b747c2e,ab0f911ffe235e01b310584be8ba7b9d9e5d3d5d5a63bce6328ec0f604057f76
+us__scenario_100__state_refundable_credits,767c9110f091d5588efe7af75dc865582582354d52c4e5a4e3608d9cd074cfdb,f8cece27dd6e6a74968f1d88d1beb57d3fd053cdcab140888d8830c30cf17e9a
+us__scenario_100__tanf,7331bc795e7e0fa009ee994da8712bdc20ca699f86061326833aa6bc68366701,10f0ff48da798ac579a6bd5cd6b286c24a7f90cc37b817fd590915bf7d72a6ab
+us__scenario_101__federal_income_tax_before_refundable_credits,2ade35296a93669322eac38ade7d98c3f4accdf7a68d2d6befb1fac8c4d4160d,02cdc31c2e97504e5d2cb812be0142f385fccb2232652ce3ce71384d529293a6
+us__scenario_101__payroll_tax,a2f08013c87417df310e5bba5659fdf8e70c3c889de68f52f01823ef05d01e09,b762fe92671cfd200d27ebe71cfa91c9f15aa388f38047b4886a727cd01681e8
+us__scenario_101__state_income_tax_before_refundable_credits,4a51ce741f9c4f5e11debc391104c9bdd78ba23a379912d224cfdd8232835533,2c03e3495e68b19fb648a5c21134b23f0f35401773765bad122d24d7f9d3c2a6
+us__scenario_102__federal_income_tax_before_refundable_credits,cb7a04b3be14992e066da0f914e0947c2c77ec5297a3b459d9f512806b7c83b0,74362c35647c123d319e9452e0b6d24b81283a326531a0107b14f13925687c71
+us__scenario_102__federal_refundable_credits,1e601d03532140264686b284f460d2116091b602a6b80a4ec30d7f3799324997,ae74d7f50b0d6de89a090d1dfdeff35fc4953aa940e1e26481f55c4698c71f54
+us__scenario_102__head_medicaid_eligible,d5d3ef6446b0a066d66bc5785940e16828da8116caa2149a1d1927dad6a94729,751f23df5475cd27a58eb4a61a8afa5c9cbd25d9c10b6dd5145119c1a4c11c51
+us__scenario_102__head_medicare_eligible,e035d850b7f1bb7e1613eb438e8df66b7425a12cd160b2a8f7c8093a49417457,e3c7848bcb73d4b112379903f96efaeb0a86c33329271f1d002b575905cd1680
+us__scenario_102__payroll_tax,f0aa4e93cce162fd3a3993f2ecbe082c0104e98030b6f271f0cb07fe5faa6eb6,d3e9fa8d9dd81ca3fcfa7f20953b40f933a7f25e09c12db278bac12da40946fa
+us__scenario_102__self_employment_tax,8657d250c36b165d52ef2648ecf991d3aa4517cae167f3a8f673b5cbc1b6e573,c20e61a529c60cfd66a8ed543ec7b930133dc678aa78d283cc6d43b1eb7adccd
+us__scenario_102__spouse_medicare_eligible,f46b896f82b5523fe3003eee2b68291323798151d9818d1b614c72404193c311,56555b70da3ddbd5bf3f844126541b7c4ec63e69b3415549dc4e9ac73da019ad
+us__scenario_102__state_income_tax_before_refundable_credits,190bffb5f3ffaa9767e94370704f5fc69540b7dac0c34e2f42b9d8272bafb2ec,87c1721247774cbb9fa7ba5f08fc32e0a0d5b7edb99b371448a4e9cb311dbeeb
+us__scenario_104__head_medicaid_eligible,2be536aa9e33fdc74fb76b8cc8ecb299bc6c6c4b0ec941b37283379e4c02507d,df565d71c1643211ff3ff2db8de166d96e7816be78baf39c115fb8696c9bd35d
+us__scenario_104__snap,ddd90e125c60e639aab3fb91979a33aac4d6de88fda09ffef7bf32d89e599952,918f3d22f7bf8d30f940c35441a4d60dae9940efea8118f52f0f5f7e7f83285e
+us__scenario_104__state_refundable_credits,e1ccbfae16f82ecc76808d1bdc19607ec277f01a04140777f2853653d7624c9b,48a4b28bcb0b63f57af2e47109f0cda96823db8e281ebbcf46675e98d8a0f605
+us__scenario_107__federal_income_tax_before_refundable_credits,ae86382325d449aa954d9ac405e3b35ed09a4f64375a3dbb9b29fa82e4b702f7,f392af050b61548f936e52455de3aeefd5f32310a0f1478d8d6d6381b054ddef
+us__scenario_107__federal_refundable_credits,8ce30c4292c1d550874c94f936ec81c91e7bda6501b4425baf686768ce12bc97,f4bebed066e53a96dc3758b7efd30b421663fb3a56019d93ca6b674097b6b0bb
+us__scenario_107__head_medicaid_eligible,d64333207d87b423bad3bae0ef46495e35857778fa0a6c60a1636ec8f557f280,22e27326f52dcbf196f98ca50d3ea3febe7691655708a4c561ab7dc331e6510d
+us__scenario_107__payroll_tax,492359a5328a9233be292b09edf7a0011778a0aa21744aa6abb5a771d4cd32ba,45a6e6efced759812f979e51794ee4447950fa7d3a99b59b2234081e1a202d8d
+us__scenario_107__snap,a8c76325a7f15ddd29f9480f8ae9a9ed424bb6c0b9be982a75296f8a78654c73,c29da5cb840d5ef24525b4df767de20d35334c4bc5e7a8e0b1c0fe17a2ae5ad3
+us__scenario_107__state_income_tax_before_refundable_credits,3ca54642b83a48e0a9c810433c9d185b443b9d4a1ad29176d771dd79d55a52ae,e4ed77a3fbefcda2d7931eb41de980c5e87f9b61f40a9b0ef77da25bf775342e
+us__scenario_108__federal_income_tax_before_refundable_credits,f055f23f31453e1f51c7f34882dd643ddb36ad83b714f8543067fd2c4c9d7b0a,99801a44d1aabbd23aa38a7d1b29d799f4260f88f37cad7feee5baeb7a62f45c
+us__scenario_108__federal_refundable_credits,1c413f0dc2dc3bb0f8b33e042c2805e0966026e9c417ee210620d33856d14446,481c7d2928c83010123aee24ffd4f4135a74a91a3ccbced7ae43aa06a42e3d81
+us__scenario_108__head_medicaid_eligible,6a483668c681e02232875b8de5c6c95917e4364a3abf41e8ecf6e7bf3448530e,206ea10d52f62a23e88779c7dc4d54dcfc037270f3d47f06760b2c1b4b7bff6c
+us__scenario_108__payroll_tax,02809809d38b37addbfd7ed5ce7ae0b06b3077448f4daae0cdbbbf5e99b00654,0de4a4b0f4a4eedeb8cfc470c02dd520a9eb33accf45c34f99ad5b22f669fb74
+us__scenario_108__snap,0d0150c16a3d7c669900d5095308d611cb11d98299427e92ca3249857c93a6fc,0a76cc9f1894dfc3f49249aed4ad5964e62a9224590ad90b1cf19cf8cd607483
+us__scenario_108__state_income_tax_before_refundable_credits,e9c0825df31e9d945d395d03ab9ebcf9afe01dfe6324c28d93a91bf7e5144071,55578f41ddae1c0e2d0efa06856197efe8c704de227a90f7d757aa77a64222f2
+us__scenario_108__state_refundable_credits,47116e66b9b1b6aaddad882ae7465a3199afb29f76e5f8f3f000a6651df02aae,ee330c536da72a7eccb9bb4f0982411c3aee2cb179155d7a6f9534cc4461cb2c
+us__scenario_109__child1_chip_eligible,95b07ff668b9575e4309589318f19b58020cce8d40172ba5de626fa839c9caff,4179653b3eb573f877ae4cd6dac9362b5baee7f96f5901830f8a367ffdc062de
+us__scenario_109__child1_medicaid_eligible,c806fea0f6a8b994fdf9205258f536acadec10c25daa12f6ca105c8e4034bcf6,4bc3938fd7c76188713048b6a01cf5ec34245afb71c38abf6bc332264fbcc4bf
+us__scenario_109__child2_chip_eligible,7e7873d309632b91ea07dcdba15c95c84e30759c00fe7be071247b983c2f9e8d,8437126d76f27b635bb201c035b1ebf73a8df5352962e49c1237d49317570180
+us__scenario_109__child2_head_start_eligible,f46b048bee44d197f028cae7290c62e42ccf3e5e5e7468331607ec3c136d4119,ff4e4d4fac95ae515a1ded072a26a93678a4fee677e318184fa805fe031d4f66
+us__scenario_109__child2_medicaid_eligible,5f41323210a69526bcef644dc26eaa0f5d4d3cab8aa7dd7adc42d0cf28e34f55,0deb8c46388d3597980907374bd27e0b487b907c9945a868f31c08202fa75d8e
+us__scenario_109__child2_wic_eligible,ff0267c6ef81551f3877368a43b5218cfa245e1194a95dff0e69ebf70d8ccb77,78f286838cd944b3e37925f586cc2f2f5f68861204f3e5dc8be6594c03a3bebd
+us__scenario_109__child3_chip_eligible,dde97cf5286042bf287f0a20fb76fdead5508c0e9a84b29c104e9bf8ed5b1fdc,b30f8127d5631222cae718b98ada52886c0a400050b5ba1cf7e3bdf1a186010e
+us__scenario_109__child3_early_head_start_eligible,2d40b21ecfddd39defe7add5b274ca5d541c7d4ba5600128b0631eabf2eff666,26176a8ad6de724f4d8c5c47e4911c55b6b4e8daca6cb6cbc98057751f501741
+us__scenario_109__child3_head_start_eligible,9bf66fdc97e155bf72a12f9b4de76f3e3ce2d76e39610e67bc6c10702f25e6da,3652c93d43605f903e1e3ae9388150d47a97f2672cd617fadc556865b92668ae
+us__scenario_109__child3_medicaid_eligible,ca2be8657ff9de56b93cb7cb0a523e8fb5c22bea9e253d94661966a0a6bde1aa,f0923debc63db7e5edf9b40de1a35df8937ce0e99d3ecc27fb60c6ef3b8c8b51
+us__scenario_109__child3_wic_eligible,af474fc81edb2a4f723848479c0f3a499f588f0081e83740d215ba59a19cc3ad,9ccf3a7e8a096d6450d84dbc2fda4136ebdf18be87515b08b2f0f23e73a134ff
+us__scenario_109__federal_income_tax_before_refundable_credits,4d8d528b04dbf78859d3da734e6c564f21c7ddfb57d6cc1cbf6c8c792771db53,be4af2f46c406bcb08c621ba20433a734d9ecdffb1a47b5a2973219c401cd9f5
+us__scenario_109__federal_refundable_credits,03620d981768afde52a1fc57d2b9b27fc864ae34026e5414b678eec8782bd6b1,c66a0a413ba04771dec8ca4612e7b87dca4b7dbc24748ec02fc58f340932c813
+us__scenario_109__free_school_meals_eligible,de534fa64d3370e65f6d7b5d407bdfaf4489cf1e71de949d28ef815da2073b76,b92f7666fda454f815d74ccfb926e9e5fac2e7ce754fe1ce9de630d979ddf3a3
+us__scenario_109__head_medicaid_eligible,e28e69e317185feff2783d9cdd3ee1a87cd2afd9d26cec04e2be619b20e454f9,4efd6c417a20ab25a4bfe8d33b538e37264cba3c8fee877377c66e04a410b8c2
+us__scenario_109__head_wic_eligible,2137dfc79fb6f44c4be90a4c29879317924f7d86111820852a78f38605790804,e9f69a0cc76f50db8b403dd46f905776815a156a2bbe7c53a50003ed22591cbe
+us__scenario_109__reduced_price_school_meals_eligible,31bf934a56c29f7f00d7c7c42f59c897b5a175a61da73e7ae5fe75fc6ecc7bcf,6d9b2b7eaaf2e304e0ad51c254f0a181099931d157140a5d55596d84ce84fa40
+us__scenario_109__self_employment_tax,0133f3dee663890285aa94fd9005cd18c50c20487feb9ea9490901e54c34cd07,38c60acdc7d7daa6dde74f177afa29c26a7fd504d2bebe77f40a0527f20ea4e0
+us__scenario_109__snap,3398a1c92ff3a807f55043816f7179feddf58e6b9af5ab452e5f997d8262b922,6e7f3cd32d2d91a2567ba2389b3d6dffd6fceae2d3ee379f0342906fa1bc5659
+us__scenario_109__spouse_medicaid_eligible,f21941a8c827445dc9ffd1d9fec946c43e531df7ecb97925148b006aa396d9aa,f1e03b0784dd96fdb9eedac1f34427f2adde5f2082fd660972d62c88f59cee88
+us__scenario_109__spouse_wic_eligible,325d38a414903be1464707ece33fe114afa63a8e00c8b52f57c34034f5924398,c5b9e14bbc394fd2f65c3b4ddf1e3d8f3c0135764e4f9125c0fe5b8a07742499
+us__scenario_110__federal_income_tax_before_refundable_credits,05988f00e685a69ba84798515a67edf1145b7837bf3284af99214bf886abbc35,caff7a4cdaa3625bd69b1412f2c8cb001b83e618ed417ee92e726ab1be56a0c0
+us__scenario_110__payroll_tax,9a3fba015b3d3d4baa3e800fedab961ff73d097bb6fd79d69e270408f8d73360,aca7268503636d6e8e25022cac73e4003be0399ce9ff4ee5a00ee973dcdaf10a
+us__scenario_110__state_income_tax_before_refundable_credits,e6aa5b6f8a2c99a36fdaf6317f1f0088bf2cbbc244d88cd736f2cdfe90a417f2,f88cd19eab0cca287e3c367694b0a6b5e627eb95b3529b7b59d4ad846bf0cdab
+us__scenario_111__federal_income_tax_before_refundable_credits,43d0ef06a041e5007925f42ac571a195b241e9637d19d0b8558604819be914b7,492337b58b95c3ed7954c8ca550c3374f2bca1fbdcc4e373d055964e51f0c9a8
+us__scenario_111__head_medicaid_eligible,796e50469600b975ca6c882bdc31e97b7d134a5910a5e610296f35e08b8f3fa7,c85f7a15f9b4bdafe60197c9d4b89d568071bdbf0b7f2d8e0fca83d95c1fcfe4
+us__scenario_111__payroll_tax,3beb0ace746fb3a63ee03e1205ad309dedd3bbd43062ea75a433d7f111dfb361,513c770f6a6cc764c39ac5c53c079f27692b9fd3f138e0bf9692c319d16e5751
+us__scenario_111__snap,b64d6497d900bdfaa8402f2f8666d20127f340800b74e293d0e51f83d6d38fb0,95a092d05068cee4aadef05c2d697174c94c9384bde66fb42058ea747074461c
+us__scenario_111__spouse_medicaid_eligible,85746ff35f782bcdd6e8caa011c15afbd7a7e521b456787d321f5b0df4be7616,d7993641cb2c7b15c57440bb948e5d76090993f247d2084297f121acb3c40ebb
+us__scenario_111__spouse_medicare_eligible,1b4acc243558ec551f071bc68f0a9e2db86c905b08c58e1405bd372e454ac526,b6c11aaa57d23f567b259433f8a4b138419cdf7161c6d5d879b457aeee823d04
+us__scenario_112__federal_income_tax_before_refundable_credits,4e724060206eff175e4504eb2caa1b272013bf1c7fca80397d6217b9f3288157,0ecc7c6bee1489e81a8eae1183d2d9b50b06f4b4da652a5873489d38a9639c95
+us__scenario_112__federal_refundable_credits,6fc1d4fb578c17ab817542e899a18492b0a02518ce31e1e522007e80eb37108d,df1c15a914ab07ed70b700910b46bfbc8dc19fd978671da23d43ec10d3ed02e0
+us__scenario_112__head_medicaid_eligible,239be8c8c005b939b31e41b63d3d9c094c6a301e60a93092e3ac7af65a2ff275,cf005069f379d417e3274be065641b60c14779d6e1817a078b5769f98c136096
+us__scenario_112__head_wic_eligible,c3991638780039d33fe9f6e3c34d50da701f0c919f0f67998e62b5b53d596411,a67037d1bf6467d0560a3fa3d2cf93a456e9ab93be0965f511a08e78413a4989
+us__scenario_112__payroll_tax,60d96684620fee48910f39733de20767413c14547b1142fb9f6c559244bc7172,62ff4ac380676683da71168ce441119a931c332f23a94ad2fe38deec48b691ef
+us__scenario_112__self_employment_tax,92f4fde0e650ff25da9e097ad237c91b7c893d29962e071cabe9ebcab71ff218,2fffa4ac34206dd5c70328cefe0d4856b80f86cf7cd71b7f441abfe294c8f3bd
+us__scenario_112__snap,120fbc66d889f0d7fff81e1ae7f7ae9cc067ef54ac77317d5f138d795103d4b6,0ac51f3915858b8697cd370dbd262a4710ab260632f587af8324d0de452032a5
+us__scenario_114__federal_income_tax_before_refundable_credits,7a85ca29cb8f93840e8923248d3a3952e963ed522f6e8fba1558bb4f45867780,ba069da56eb141ae8fa08c0acd7b72db4018c9d59f2dc525fe3e2eae87e2f6b6
+us__scenario_114__federal_refundable_credits,4893ee696228cb54f450e9dc6c27be2f269016891f0fb00d235926b36107118d,67a85e18b6df494d0bea9ea6322784dde7da0013dd82731b1ec49d2b48906f66
+us__scenario_114__payroll_tax,728ce80c383ebea347b59f6b25ac22a735cea327de9586aae7a1f47047870fb6,61c5b2d82967d3af7df146d7ea4412e0007c2937109a90087177aea3b7f9f9d4
+us__scenario_114__self_employment_tax,976e0494715515829e2ad02f305cc25016077051b55baec3eee1377956c3e8a4,b28a5d2fb6758654353e8be605f64d0a843182a0a08e65b3595e4fe63be5e5c1
+us__scenario_114__state_income_tax_before_refundable_credits,0ec84f3fb2cdfbe024118f9e95566f1db017eed02b82701334b85b0c8e29fa5c,4f07d313983947636f0243f0d1c4dee09517e8f783112c64b898557ff55ecec4
+us__scenario_115__head_medicaid_eligible,ecce7759f91e86e416e3790ebd94fd8398dd43380b496474d0517ff5ac21990d,02f548c330509d4f280a34d9e2f7b2b18dfc6b767dce5a51e0e17865eebdfbce
+us__scenario_115__snap,634c417adf074bdd41cf1d4360f098e38645492ca5e486e05ffd1764bf24f9ec,3ede57fcbd96772eda588d55904edef6c1fcd3fd4c276c5e86df23367026ceb2
+us__scenario_115__state_income_tax_before_refundable_credits,94b4a8cceaeb333aab9d4cc4b578e4f2f5a2e8a201d73159eb92adce77a7660c,2de376c13fa48682cfeedc791b67fe9589e72e5eaec867434196950c4d04caa9
+us__scenario_116__snap,d875ad46611cd5214dfaa9c96d4a3a8821334ab2c97a93394607f17b279f7d8f,6971485bb9b884f5db2b4c85b8031d6f02b12efab642ee87a46ddd72a24b436b
+us__scenario_116__spouse_medicaid_eligible,14ede8f372d1d4ccc74d4a617692f663b62de35a102b7767aaf6cf4a10a8c7d2,53e47fbabb0aaaf5ad4aad7042e5a579fa726331c69993a8275635d5e6ca3f2e
+us__scenario_116__spouse_medicare_eligible,0efd2eec8758157d1f455e443b0f698c82bc22bce2698af842aedd4040c08641,5d953ccfb4800970e84f8a0c3ac761057cddb400bfe906d65761c0600b70d6c1
+us__scenario_117__child3_early_head_start_eligible,0926d12e2f0241120078e26167ad9cf017333dcb4b8955a2d5a09bf892c007b2,fb8a3a1900d19278385a97eb392b00206f4d61e82f090c754abb697565cb20a1
+us__scenario_117__child3_head_start_eligible,44703423e85c5569457e69aa31ef80b2659065d6f27fe639e083fbebf58fe5ab,8c0858c6f18e6a21d8b2e443a088a19d25ab0b0c09c0aa10aa25a40a0271335f
+us__scenario_117__child3_wic_eligible,31ff794afa1cb3ddb82ead5b6dece0a9b998b02ff644f1214d827f7b6a90d20b,4c82af05c92393e1707d1699f64d3d02619c4a506d76d10ba2ab4e9d4f4165f1
+us__scenario_117__federal_income_tax_before_refundable_credits,4a6361205046f5e398ac79536ded8426233a080b9e59d0b37df42b91014e2228,9ea96545c3e7299ad6b0cbe5e66c861129a3506dfc8b2d02b8691ca1d15c79be
+us__scenario_117__federal_refundable_credits,b0f1d7553d95d493cb21d918e67ffda978fea56f3db7a5f7c6a48ea50230606b,e7d2a782ec4c5074b9650ede908b9a02eb5cb838fddf78745e839b539a6b312c
+us__scenario_117__head_wic_eligible,d65f67cf6634c591eee1aa2dca4f3abb6eab181efbfa9683cd69121f74d78ba3,999b453c4ea5381c95feea026b861503c86c28346fea8abcefb8e3f65a6bc3cd
+us__scenario_117__payroll_tax,5723db3827329d4163c1cd858383de1c8f4dc2b7338a45037a2a28ef07b22551,d861a931661c0a076d8ebcb5f3b478231c9e42f94123c61f4135c8dcd41c1bc1
+us__scenario_117__spouse_wic_eligible,3de7fecf4f8871a549d78944fc9fa629faad351400955f029091211d2cef52a5,d3518137770e1e43fafb36317e5a4dd1224f90d663bc6e7f03944820d34a09f9
+us__scenario_117__state_income_tax_before_refundable_credits,e84c1f2b0574b8a69dbf50a8e3af3c45d3505f8b0025262fba91e0cd4ee684a7,e03da95c4a5b3c6ebdb4e925536b66e6d71825da755e938fdc2c548fd19572ac
+us__scenario_118__head_medicaid_eligible,c73817e22dd1584f2bbe2a1f09bca6927748893b5f37a7bede5423cd1acf6f0d,bfb90fee56f8afa66629351f04798ed6818d754c832047dd445170445285a949
+us__scenario_118__payroll_tax,ac99d20fe15a935dd5b988b6dcac640b33c3845851e44786a260727e14f52c85,d6aaec3987c5199e4297fc2d723d03dd240caba24a4f9581a1fbe2c170d6f0fd
+us__scenario_118__snap,d49c1b2caaa556cdb03e5bc0c4bba73d0764b0002dc6f81c41b8b5cfc6a78c3b,d66258f2f82c7aa795382a0309da577f83721196b65f3c894fcbdb47eea3de9a
+us__scenario_118__ssi,519f876cbcaf7c9f8c420b205d6b9688cc92e190e7d2667b4de0113cf33960ca,b9ba12382a09c64ed7b38e1b7847dafd98cade0601e969a273b9c4a974244689
+us__scenario_118__state_refundable_credits,79e17fe7e5c8890456963999a6499527189a638093411719e7acd7460f20f891,1fa0ae81eafc8fe63ee68ee99f8b98428f4805b51499f500dc38b68bcd10c231
+us__scenario_119__child1_chip_eligible,380eef556ace7ef7f6bc5d713e5843dd735ad306514c576ff1057355704ad129,17ecdeb200e5875fce56e5e12f275c3330e9e62b756e4953dc2ed5069928886f
+us__scenario_119__child1_medicaid_eligible,f668ff1e7ae90feca959db659a3ae7454694f0086f0cf96c21aa48fbcd3c37e8,d513b32caf26de5ce33d49f27064fd8be3795cf85bab12eb5a7c2ed57c7d4ecf
+us__scenario_119__child2_chip_eligible,a9b1e54d772c0f72bd23e442ebcb3e4b2c22424dd8e4b30ffdef6359a3ef7d26,d3145c69dc8724c0cbfca3788c092cc91871ff062710a5159c697e1e49979ab8
+us__scenario_119__child2_medicaid_eligible,7cb313e5e8fb5d9873baafe18f7267f69f0fa26462027805052b82945c94b108,1186a23be3c19588241bfe8570c02e655f77fcb020fb966268ed6272dacfb313
+us__scenario_119__federal_income_tax_before_refundable_credits,68f748ccbc0a2fc3a151aa72a49c5df3513a500d7daab36adb6fc6f2384abf06,483b63d5796504255a16be834b8d108b39917f05126887cdeb0a6d581eb8e86a
+us__scenario_119__federal_refundable_credits,3645ecc66367d998aed76a858e68707eb8d741bf5271f9098cac9f13dc0543ac,26d605e9ff17a841134304da4101f9fe35f1eb3e2a5592edba481181428c621e
+us__scenario_119__payroll_tax,2e3e6e070681e314886ab9258066fffb00046304cf84cf51d7c7498a5412ef6f,e5294797ad9fd67690c33e56af234abe913eba73bb9abb5701f49a53467ac8b0
+us__scenario_119__reduced_price_school_meals_eligible,71c358e735d0e5b84b1fce88b1b2c6e2fd2a4a7aae846d4ffd6292a0bda5f688,5ef0c626bbd708c1ff5dae3ac9d239f8441037088dc638f482ce246c6ff6e556
+us__scenario_119__state_income_tax_before_refundable_credits,ff172164067e0a7652f30b1bf2e9a56f6a0b5670b023d574f0d4682f179eb2cd,70f9a942a452de66b585596ad26475889eb5bddb6e18da5e030b8e5eec449777
+us__scenario_119__state_refundable_credits,c9054d4b0d62e0ff8c80ef82f83f04616f35a2225e69fb51135c74de939ed838,20f73fb637134300b1c7d3a74ecec6ad9a4e2544163308c856e4d6c153c20603
+us__scenario_120__federal_income_tax_before_refundable_credits,ea6a0aee8c1c2128c2964ba760fdc2677b5f9eb2b24e92bd02ae513f8eaaf135,e12ff6ac781031ad2a409b596a002f99f37952dc964294083ccadb226859ffa5
+us__scenario_120__head_medicare_eligible,38423457f0d2f776538ce275d1e8d4614066e17bd7c163adcaf72765980cc522,eeb633fc78773bcd0efed8dd71893b20fa0213a5605078c0a2c01eaa9bb128c8
+us__scenario_120__payroll_tax,22f4bd0104c2e38801c02e15a04ae201774016675f385cef9d6d202fe0e532a6,4fdaace099309e80618dff547161684de344bfdbf11876f3709197ee21d7f70e
+us__scenario_120__state_income_tax_before_refundable_credits,a042b5e45360fd78baf307cc54b94baacb2e3cb6484b6915e03e4626aef13ad0,ba4d1efeadeb0b01116b1baf300f52d2e954239604677f89f201e94dead6a148
+us__scenario_121__federal_income_tax_before_refundable_credits,61bfca59d67cc2550a0886929d585cd61b021edbf52e10b02a116c6eab77cf70,ec63cc2614dabc2731ae4cd4c2aaf51ff02a2bdfe46e944b673d4d3db7425b36
+us__scenario_121__federal_refundable_credits,21245c794e7489d82d006c47eb91924e7adf2641c548f97a077a37fd96152dfd,172a7cc3c7dc2766ed07f83ffd2fd74afa82fa4903585a1973982dc6fef8e94c
+us__scenario_121__head_medicaid_eligible,f1d2f12bbacc579e157d86e1c3ba16e03e842336293472f8d256c129adb3b4b9,72fbf4afa7aaf9bc3d5898710bac6ea7e3d7a5effb3fa7270abecb173b859f31
+us__scenario_121__head_medicare_eligible,30e015f3c46d6e5083eb214a17bde851abdeb42c68874f9786caef816b0bc938,8cef3435d06dc6ae0532f2b09b4cdb9a08690a1039a1bcb4495493fd262b055e
+us__scenario_121__payroll_tax,640422209d50a59709a5d256d27a53027530bcc184a59d09704b2abe58ad7d09,66b09cf08016c1dc3370ec43fede9a05b96ebc5e009ae2756e067fcf1aa6e9db
+us__scenario_121__snap,03512911eb78f3468bfaada9c89697f192ab69aad5e3e2b0dc0501a6de854b51,c27267ef789ef6c54f0a9673c990e2555ccd908b694d96d06d125f1ab6da2aed
+us__scenario_121__state_income_tax_before_refundable_credits,ce60fe57a06785eb16d777ea41b1cd3c9a76e1982937a20d627a5a68c819c61f,a8b657c3e9398e752ec8ab3cf4d875ba203398c56ba07803b1a6b0ffc7533375
+us__scenario_122__federal_income_tax_before_refundable_credits,1651e5d2391184bd11a1c8daa6b35992e02f055052b9585172fac9142b3c3be1,8dbaf3a0688ebc840977f814d334aa59e4ddc6a4469af9be8e002f3fc321ea85
+us__scenario_122__head_medicaid_eligible,9cb1915eea284a648754090f09defe9de9174b415c189f7fbe36e396b2cc153f,f33e1094f352c7a871ac9bddd720e1a9cf13c42735d4cbcc766bb5775599be76
+us__scenario_122__payroll_tax,6194402baac22b4f36529d9b2ff9da2c38282f5954eec05586484ffb93af6ba9,8b7cfde758ff2f97f0bc27f7eda3f2326f9a04ff7ca3d277ef9ff936dbed67b3
+us__scenario_122__state_income_tax_before_refundable_credits,a72141faa7a7768a517b740055c76e94e7c354c933985d22b451327d8efa6276,445f3932b860e766e8b5085e2ec2d67635384770efeeb9888768b69517e86f29
+us__scenario_122__state_refundable_credits,b032c5c37245f6c6c08e03b129af3c18af4f5095260eb2315c74fae180986175,d04072605a758bf7afdef0d7bd603251a3594b41f84b4d7c4d34118c1787cf0b
+us__scenario_123__child1_chip_eligible,396ea9294f25df4757accd3fa501d5116ab243cfd339fa977f462450b459067d,655d730efd8a49a87d1448fd3a55ff88b9872ab3bd5b72cf5450350f7bc8ad23
+us__scenario_123__federal_income_tax_before_refundable_credits,e77fcb1a75fe68eb875554162b028ae98119d40495f36fe018834c489d35db55,dfd9984399068d514a134fdfbd1dc5a7513e5b9e3211f0178fc10e72ea178f9d
+us__scenario_123__federal_refundable_credits,1b4e0478dd57381352d13443a3bb844989a26c0843514ebb45d2a142d0653232,a9abcef52c7be874497616bf1ce520817ca82b28aadba529c84c2bffd47fa435
+us__scenario_123__head_medicaid_eligible,bcff1a25f1f72e5213b3c6611ef16ca31b2f8bf6f89a2fa9376e171df6866671,746a18bf8fd8d6650ea14f83587282a4119c5a7a4e053619f18a86b11b82f84c
+us__scenario_123__local_income_tax,73d046ea1cd007f5d7371adfc9df36cc8ef4abb512879b2d82ec9d6885d75bc3,a5aa6a1188d8fda5e8d7872e1724e0403e7114749df22a94769458369a2b5e6d
+us__scenario_123__payroll_tax,bdfa2aab60f42aaa803c2a2bf0deb2a4afab3f5edab46391e592ca7908d8c298,5cb7217c4b2d00506c5af57448942416f137f84d588a7401303702815464328b
+us__scenario_123__spouse_medicaid_eligible,0ea7b3c8cc4c093da15503cb7cfaf83494a83b12184a598bd2138df85b630449,40d4250a45815d85b6550450c375aa9390128d15b3c74eddad4687365e5ab3cf
+us__scenario_123__state_income_tax_before_refundable_credits,8d451456e9eb8a8a108d3182cd1eae9881a9166a1172b52fb5419f3ea49ef9f3,df02301c7b23696cd23979446c4e87c8caf9d0d722a0ad491a135191b2010398
diff --git a/docs/haiku55/serving_probes.json b/docs/haiku55/serving_probes.json
new file mode 100644
index 00000000..3f496bdc
--- /dev/null
+++ b/docs/haiku55/serving_probes.json
@@ -0,0 +1,57 @@
+{
+ "note": "The two probes behind Claude Haiku 5.5's serving claims (its model card's notes and the release note): the request settings, HTTP status, response id, the response's content block types and its usage, from results/local/haiku55/haiku55_thinking_default_probe.json and haiku55_forced_tool_probe.json in the main clone (run 2026-10-08 during onboarding; each source file's sha256 is kept). Thinking text and signatures are left out.",
+ "probes": {
+ "no_thinking_parameter": {
+ "source": "haiku55_thinking_default_probe.json",
+ "source_sha256": "b4e9e415c2867571c63a493c1bdcb14d299a8f3f1195ccf740f6aa15607ad6d0",
+ "probe": "one question, no thinking parameter, no tools (does the default request think?)",
+ "request": {
+ "model": "claude-haiku-5-5",
+ "max_tokens": 2048,
+ "has_thinking_parameter": false,
+ "tools": [],
+ "tool_choice": null
+ },
+ "status": 200,
+ "response_id": "msg_011CfpxFX7LMQXwhoCSjpP9H",
+ "content_block_types": [
+ "thinking",
+ "text"
+ ],
+ "stop_reason": "end_turn",
+ "usage": {
+ "input_tokens": 54,
+ "output_tokens": 123,
+ "thinking_tokens": 119
+ }
+ },
+ "forced_tool": {
+ "source": "haiku55_forced_tool_probe.json",
+ "source_sha256": "4b2ec49ff811e17aa40237b18f3b9eb80db7118686b33ef4912b45ef0c01dea9",
+ "probe": "forced tool_choice type tool, no thinking parameter",
+ "request": {
+ "model": "claude-haiku-5-5",
+ "max_tokens": 2048,
+ "has_thinking_parameter": false,
+ "tools": [
+ "answer"
+ ],
+ "tool_choice": {
+ "type": "tool",
+ "name": "answer"
+ }
+ },
+ "status": 200,
+ "response_id": "msg_011CfpxFbsq3B3zCWjJBgNrh",
+ "content_block_types": [
+ "tool_use"
+ ],
+ "stop_reason": "tool_use",
+ "usage": {
+ "input_tokens": 529,
+ "output_tokens": 33,
+ "thinking_tokens": 0
+ }
+ }
+ }
+}
diff --git a/docs/haiku55/spec.json b/docs/haiku55/spec.json
new file mode 100644
index 00000000..4c308b75
--- /dev/null
+++ b/docs/haiku55/spec.json
@@ -0,0 +1,238 @@
+{
+ "note": "Inputs of the release after dashboard-data-20261006, read by scripts/finish_haiku55.py. The release folds Claude Haiku 5.5 onto release dashboard-data-20261006 and installs Max's rulings of 2026-10-06: d1022 (the reference adversary's eight confirmed or ambiguous cells) and d994 (the two Louisiana cells). Ten outputs leave scoring for every model; the references, the incumbents' predictions and every earlier exclusion's value stay as they were.",
+ "release_tag": "dashboard-data-20261010",
+ "base_tag": "dashboard-data-20261006",
+ "base_sha256": "aa34e5c9ea926848dc7af460a98f17021bc42dae54a8015c9988a85e79a5d462",
+ "decided_on": "2026-10-06",
+ "rulings": {
+ "d1022": "exclude the reference-adversary's confirmed cells: engine defects in scenario_018 (AZ) and scenario_025 (OH) state income tax and scenario_043 (CO) and scenario_082 (NY) state refundable credits, and the dependent-return scope cells in scenario_123 (PA) and scenario_093 (MO) federal and state income tax; regenerate the four defect cells in a later release once fixed policyengine-us versions land (PolicyEngine/policyengine-us#9928, #9925, #9946, #9948); scenario_026 refuted (Max 2026-10-06, 'approve')",
+ "d994": "exclude scenario_051 and scenario_077 Louisiana state income tax; keep the published-amounts convention (no change to Idaho 076, SNAP 008/038/109, Maryland 068); version 2 states indexed amounts in the prompt (Max 2026-10-06, 'approve')",
+ "d1029": "open (household scope for version 2); the d1022 exclusions of scenario_123 and scenario_093 stand until it is decided",
+ "2026-10-09": "move the references to the policyengine-us release that fixes the engine defects behind PolicyBench's exclusions before the Claude Haiku 5.5 launch, and regenerate the cells it fixes instead of excluding them (Max 2026-10-09, to the US + core hub: 'ok lets try to fix these all today before the haiku launch', then 'yes i want to wait for hte fixed engine'); this brings d1022's later regeneration of the four defect cells into this release"
+ },
+ "proposals": [
+ {
+ "path": "reference_audit/2026-10-05-reference-adversary/proposed_changes.json",
+ "sha256": "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf",
+ "records": "root_causes.*.exclusions",
+ "outputs": [
+ [
+ "scenario_018",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_025",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_043",
+ "state_refundable_credits"
+ ],
+ [
+ "scenario_082",
+ "state_refundable_credits"
+ ],
+ [
+ "scenario_093",
+ "federal_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_093",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_123",
+ "federal_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_123",
+ "state_income_tax_before_refundable_credits"
+ ]
+ ]
+ },
+ {
+ "path": "reference_audit/2026-10-05-louisiana/proposed_exclusions.json",
+ "sha256": "a78120cd71110e823b0d7dafcffc5582d300408491335eb935e02112a5702545",
+ "records": "exclusions",
+ "outputs": [
+ [
+ "scenario_051",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_077",
+ "state_income_tax_before_refundable_credits"
+ ]
+ ]
+ }
+ ],
+ "derivation_insert": {
+ "before": " On 2026-09-29 the references moved to policyengine-us 2.15.17",
+ "text": " On 2026-10-06 ten more outputs were excluded on review of release dashboard-data-20261006 (reference_audit/2026-10-05-reference-adversary and reference_audit/2026-10-05-louisiana); their records are computed on policyengine-us 2.15.17, whose values are the references."
+ },
+ "restated_adjudications": [
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits",
+ "prior": "regenerated on 2026-09-22 (capital gain distributions; PolicyEngine/policyengine-us#8839), and scored until this release"
+ }
+ ],
+ "adjudications": [
+ {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "reference_engine_defect",
+ "reference_verdict": "engine_defect",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "reasoning": "The frozen reference is 1,146.05; the corrected value is 1,137.30. Arizona's 2026 single standard deduction is $16,100: HB 4168 set $15,750 subject to A.R.S. 43-1041(H), which indexes it as the federal basic standard deduction is indexed (Rev. Proc. 2025-32). policyengine-us 2.15.17 carries $15,750 into 2026 unindexed, because the uprating sits on a parent parameter node that policyengine-core does not propagate to the filing-status children. Found by the reference adversary on 2026-10-05 and verified independently (reference_audit/2026-10-05-reference-adversary/verification/independent/az_018.md)."
+ },
+ {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "reference_engine_defect",
+ "reference_verdict": "engine_defect",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "reasoning": "The frozen reference is 1,921.57; the corrected value is 1,916.60. Ohio deducts unreimbursed medical care above 7.5% of federal AGI, with medical care defined by IRC 213(d), which includes insurance premiums (R.C. 5747.01(A)(10)(b)-(c)). policyengine-us 2.15.17 counts premiums only for a person who is Medicare-eligible and has no employer-paid plan, where the form's worksheet line 3 reads 'Medicare or an employer-paid health care plan', so this filer's $6,500 of after-tax premiums get no credit. Found by the reference adversary on 2026-10-05 and verified independently (verification/independent/oh_025.md)."
+ },
+ {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "adjudicated_failure_source": "reference_engine_defect",
+ "reference_verdict": "engine_defect",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "reasoning": "The frozen reference is 19.00; the corrected value is 0.00. C.R.S. 39-22-2003(2) allows Colorado's sales tax refund for a tax year only if the fiscal year ending in it had excess state revenues; every forecast published before the freeze put FY 2025-26 below the Referendum C cap, with a tax year 2026 refund of $0 in every tier. policyengine-us 2.15.17 pays the six-tier refund without that condition. Found by the reference adversary on 2026-10-05 and verified independently (verification/independent/co_043.md)."
+ },
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "reference_later_law",
+ "reference_verdict": "later_law",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "reasoning": "The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana)."
+ },
+ {
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "reference_later_law",
+ "reference_verdict": "later_law",
+ "adjudicated_failure_subtype": "taxable_income_or_deductions",
+ "reasoning": "The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana)."
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "adjudicated_failure_source": "reference_engine_defect",
+ "reference_verdict": "engine_defect",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "reasoning": "The frozen reference is 667.00; the corrected value is 1,187.61. For tax years from 2026, N.Y. Tax Law 606(c-2) (L.2026, ch. 59, Part A) gives a refundable credit of the applicable percentage of qualifying care expenses, and 606(c)(1) limits the old formula to earlier years. policyengine-us 2.15.17 still computes the 2026 credit as a percentage of the federal credit. Found by the reference adversary on 2026-10-05 and verified independently (verification/independent/ny_082.md)."
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "reasoning": "The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2."
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "reasoning": "The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2."
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "reasoning": "The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2."
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "state_income_tax_before_refundable_credits",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "reasoning": "The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2."
+ }
+ ],
+ "snapshot_date": "2026-10-10",
+ "record_edits": [
+ {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "field": "upstream",
+ "replace": "to be filed",
+ "with": "PolicyEngine/policyengine-us#9928",
+ "source": "merged 2026-10-09; regenerating the reference waits for a release with an engine bump (d1022)"
+ },
+ {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "field": "upstream",
+ "replace": "to be filed",
+ "with": "PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925",
+ "source": "#9925 (merged 2026-10-09) fixes a related double count; #10020 (open) counts the premiums this record names"
+ },
+ {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "field": "upstream",
+ "replace": "to be filed",
+ "with": "PolicyEngine/policyengine-us#9946",
+ "source": "merged 2026-10-09; regenerating the reference waits for a release with an engine bump (d1022)"
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "field": "upstream",
+ "replace": "to be filed",
+ "with": "PolicyEngine/policyengine-us#9948",
+ "source": "merged 2026-10-09; regenerating the reference waits for a release with an engine bump (d1022)"
+ }
+ ],
+ "adjudications_written_on": "2026-10-09",
+ "regenerated_by_upgrade": {
+ "note": "With --step install-references, the ruled records the build regenerates: d1022's four engine-defect cells, each fixed upstream (AZ 018 PolicyEngine/policyengine-us#9928, OH 025 #10020, CO 043 #9946, NY 082 #9948). Every other ruled record stays excluded. Read only with an upgrade installed.",
+ "outputs": [
+ [
+ "scenario_018",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_025",
+ "state_income_tax_before_refundable_credits"
+ ],
+ [
+ "scenario_043",
+ "state_refundable_credits"
+ ],
+ [
+ "scenario_082",
+ "state_refundable_credits"
+ ]
+ ]
+ },
+ "upgrade_adjudications": [
+ {
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state."
+ },
+ {
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "reference_verdict": "unlisted_input",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state."
+ }
+ ],
+ "upgrade_adjudications_written_on": "2026-10-10"
+}
diff --git a/docs/paper.md b/docs/paper.md
index d66e6ef8..8a70c207 100644
--- a/docs/paper.md
+++ b/docs/paper.md
@@ -52,7 +52,7 @@ The manifest at `paper/snapshot/20260501/manifest.json` lists:
pinned release URL, byte count, and sha256 for each dashboard payload
- `rendered_paper_artifacts`, with the rendered PDF and web bundle hashes
- `reference_output_refresh`, with the policyengine-us version that computes
- each scored reference (2.15.17 in this snapshot) and the policyengine.py
+ each scored reference (2.38.6 in this snapshot) and the policyengine.py
version it records for provenance. Its dataset fields name the reference
runtime's default dataset (build populace-us-2024-spm-20260915, from the
policyengine.py 6.1.2 bundle), which reference computation does not read:
@@ -67,15 +67,16 @@ The manifest at `paper/snapshot/20260501/manifest.json` lists:
- `population_weight_artifact`, with the committed scoring-weight path and
sha256
- `audit_annotation_artifacts`, with the row and case audit file hashes. The
- frozen annotations cover 7,527 scored rows whose legacy threshold score is
- below 1: 7,523 of 7,523 exact-match misses and 4 exact hits. Another 2,072
+ frozen annotations cover 8,001 scored rows whose legacy threshold score is
+ below 1: 7,997 of 7,997 exact-match misses and 4 exact hits. Another 2,138
scored rows have bounded score below 100 but were outside that selection and
are not annotated. A judge_provenance block tallies which judge model produced
each case verdict.
- `reference_exclusions`, with the outputs removed from scoring for every model
- because their reference rests on a reference-engine defect not fixed upstream
- or depends on an input the prompt never states (64 in this snapshot; 1,920 of
- 1,984 outputs per model are scored), the file's hash, and the tallies by
+ because their reference rests on a reference-engine defect not fixed upstream,
+ depends on an input the prompt never states, or rests on an amount published
+ after the reference freeze (58 in this snapshot; 1,926 of 1,984 outputs per
+ model are scored), the file's hash, and the tallies by
unlisted input and by engine-defect root cause.
- `reproducibility_notes`, covering model-alias instability and what is not
retained locally (LiteLLM cache, since it is a generated request cache)
diff --git a/paper/index.qmd b/paper/index.qmd
index 6409d64b..447d714e 100644
--- a/paper/index.qmd
+++ b/paper/index.qmd
@@ -891,8 +891,9 @@ reference_audit = pd.DataFrame(
["US state income tax and credits (September 2026 wave)", "Engine defects confirmed in New Jersey worker contributions, Wisconsin's elected retirement exclusion, Wisconsin's capital gain exclusion (which leaves out capital gain distributions), Idaho's health-premium subtraction, California itemized-deduction conformity, and Massachusetts dividends taxed without the short-term capital loss offset; affected outputs excluded. The CalEITC's missing second lookup at adjusted gross income and New York's renter cap were fixed upstream after the freeze; the affected references are regenerated with the fixes. Wisconsin homestead income turned on the unlisted type of survivor benefits (excluded as an unlisted input). The New Jersey EITC, New York real property tax credit and Idaho child tax credit flags dissolved against the statute (Idaho's credit expired after 2025)."],
["US SNAP, SSI and CHIP (September 2026 wave)", "Whether listed mortgage interest is on the occupied home, unlisted hours worked, survivor-benefit taxability, and whether an adult tax dependent is the filers' child (three Medicaid outputs) excluded as unlisted inputs. SNAP October-December references regenerated with the FY2026 figures, the last published before the freeze. SNAP references regenerated for every month of 2026 with the engine's SNAP rounding defects fixed as policyengine-us fixed them after the freeze (cents in the allotment, an unrounded minimum benefit, floored net income, unrounded income standards). One Michigan SNAP reference regenerated with the fix for the SNAP child support option, whose engine values carried the opposite meaning (Michigan deducts child support from net income, and the engine excluded it from gross income; fixed upstream in policyengine-us #9586 on 2026-09-24). The heat-and-eat utility allowance granted without an elderly or disabled member, which P.L. 119-21 ended, excluded as an engine defect. The CHIP flag dissolved: employer coverage bars CHIP."],
["US projected 2026 parameters (September 2026 wave)", "No judge flag: a recomputation with every projected 2026 value held found references resting on price-index projections. Wisconsin, Minnesota, Michigan and Missouri amounts and Maryland's withholding allowance had been published before the freeze; California's indexed amounts, the IRS sales tax tables, Idaho's threshold and Maryland's return deduction had not. Scored references regenerated with the published or last-published amounts; Wisconsin's reach only the corrected values of Wisconsin outputs excluded for engine defects."],
- [f"US references on policyengine-us {r.policyengine_us_version} (September 2026 upgrade)", f"Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us {r.previous_policyengine_us_version} and {r.policyengine_us_version} to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes {r.engine_upgrade_scored_change_count} scored references, for law published before the freeze that {r.previous_policyengine_us_version} did not encode or misapplied: New Jersey's 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona's 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York's Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, {r.engine_upgrade_new_exclusion_count} federal income tax outputs that depend on an unlisted input: {r.policyengine_us_version} counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the {r.engine_upgrade_rechecked_count} excluded outputs that move; all stay excluded. The re-review of a California household's excluded SNAP output flagged its head's Medicaid eligibility, which turns on the same unlisted input, SSI's disability criterion, through the Working Disabled Program; PolicyBench excluded it."],
- ["US federal and Virginia income tax (October 2026 review)", f"No judge flag: a review of release dashboard-data-20260930 swept every reference on policyengine-us {r.policyengine_us_version} under readings of two inputs no prompt states. The state income tax part of the federal state and local tax deduction is policyengine-us's formula estimate of state income tax withheld on federal adjusted gross income; read as the household paying its 2026 liability during the year, or paying none, {r.review_sweep_newly_excluded_count_word('salt_withholding')} scored federal income tax outputs move, and PolicyBench excluded them; the readings also move {r.review_sweep_already_excluded_count_word('salt_withholding')} federal outputs already excluded. policyengine-us treats every Medicare-eligible person as enrolled and counts a modeled Part B premium as a medical expense; without it, {r.review_sweep_moved_household_count_word('medicare_part_b')} household's federal and Virginia income tax move, and both are excluded."],
+ [f"US references on policyengine-us {r.september_upgrade.engine_version} (September 2026 upgrade)", f"Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us {r.september_upgrade.previous_engine_version} and {r.september_upgrade.engine_version} to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes {r.september_upgrade.scored_change_count} scored references, for law published before the freeze that {r.september_upgrade.previous_engine_version} did not encode or misapplied: New Jersey's 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona's 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York's Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, {r.september_upgrade.new_exclusion_count} federal income tax outputs that depend on an unlisted input: {r.september_upgrade.engine_version} counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the {r.september_upgrade.rechecked_count} excluded outputs that move; all stay excluded. The re-review of a California household's excluded SNAP output flagged its head's Medicaid eligibility, which turns on the same unlisted input, SSI's disability criterion, through the Working Disabled Program; PolicyBench excluded it."],
+ *([[f"US references on policyengine-us {r.policyengine_us_version} (October 2026 upgrade)", f"The move from policyengine-us {r.previous_policyengine_us_version} fixes the engine defects behind {r.engine_upgrade_restored_count} outputs ({r.engine_upgrade_restored_count - r.engine_upgrade_restored_ruled_count} that earlier releases excluded return to scoring at the new version's values; the {r.engine_upgrade_restored_ruled_count} ruled on {r.ruling_date} while still scored stay scored) and changes {r.engine_upgrade_scored_change_count} scored {'reference' if r.engine_upgrade_scored_change_count == 1 else 'references'} beyond the dollar tolerance; {r.engine_upgrade_new_exclusion_count} {'output leaves' if r.engine_upgrade_new_exclusion_count == 1 else 'outputs leave'} scoring. PolicyBench re-reviewed the {r.engine_upgrade_rechecked_count} excluded outputs whose values move; all stay excluded. {r.engine_upgrade_restored_sentence} {r.engine_upgrade_scored_change_sentence} {r.engine_upgrade_new_exclusion_sentence}"]] if r.engine_upgrade_count > 1 else []),
+ ["US federal and Virginia income tax (October 2026 review)", f"No judge flag: a review of release dashboard-data-20260930 swept every reference on policyengine-us {r.review_exclusion_engine_version} under readings of two inputs no prompt states. The state income tax part of the federal state and local tax deduction is policyengine-us's formula estimate of state income tax withheld on federal adjusted gross income; read as the household paying its 2026 liability during the year, or paying none, {r.review_sweep_newly_excluded_count_word('salt_withholding')} scored federal income tax outputs move, and PolicyBench excluded them; the readings also move {r.review_sweep_already_excluded_count_word('salt_withholding')} federal outputs already excluded. policyengine-us treats every Medicare-eligible person as enrolled and counts a modeled Part B premium as a medical expense; without it, {r.review_sweep_moved_household_count_word('medicare_part_b')} household's federal and Virginia income tax move, and both are excluded."],
],
columns=["Discrepancy class", "Review outcome"],
)
@@ -969,7 +970,7 @@ conclusion_snapshot_summary = country_top_sentence(us_top_model)
## Abstract {.unnumbered}
-PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine `{python} r.dataset_label` microdataset and scoring answers against reference outputs from policyengine-us `{python} r.policyengine_us_version`. It evaluates `{python} r.n_models_fmt` frontier models on `{python} r.n_households_fmt` households across `{python} r.n_output_groups_fmt` output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: `{python} r.zero_share_pct_fmt`% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of `{python} r.always_zero_exact_fmt`% weighted exact by `{python} r.top_exact_margin_fmt` points (Section 3.1 details the construction). The manuscript snapshot is a `{python} r.n_households_fmt`-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. `{python} abstract_snapshot_summary` Within model families, newer is not always stronger: Claude Opus 4.8 scores `{python} r.opus48_score_fmt`% exact, below the `{python} r.opus47_score_fmt`% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That universe contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on `{python} r.reference_rebuilt_date` with policyengine-us `{python} r.policyengine_us_version`, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the `{python} r.engine_defect_exclusion_count` outputs whose references rest on engine defects the audits found and upstream has not fixed, and the `{python} r.unlisted_input_exclusion_count` whose references depend on an input the prompt does not state. The `{python} r.parse_contract_failure_pct_fmt`% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at .
+PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine `{python} r.dataset_label` microdataset and scoring answers against reference outputs from policyengine-us `{python} r.policyengine_us_version`. It evaluates `{python} r.n_models_fmt` frontier models on `{python} r.n_households_fmt` households across `{python} r.n_output_groups_fmt` output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: `{python} r.zero_share_pct_fmt`% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of `{python} r.always_zero_exact_fmt`% weighted exact by `{python} r.top_exact_margin_fmt` points (Section 3.1 details the construction). The manuscript snapshot is a `{python} r.n_households_fmt`-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. `{python} abstract_snapshot_summary` Within model families, newer is not always stronger: Claude Opus 4.8 scores `{python} r.opus48_score_fmt`% exact, below the `{python} r.opus47_score_fmt`% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That universe contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on `{python} r.reference_rebuilt_date` with policyengine-us `{python} r.policyengine_us_version`, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the `{python} r.engine_defect_present_count` outputs whose references rest on engine defects the audits found and upstream has not fixed, the `{python} r.unstated_input_exclusion_total` whose references depend on an input the prompt does not state, and `{python} r.later_law_exclusion_count_word` whose references rest on an amount a state published after the freeze. The `{python} r.parse_contract_failure_pct_fmt`% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at .
## Introduction
@@ -1076,19 +1077,23 @@ PolicyBench treats PolicyEngine outputs as benchmark reference outputs, not as a
This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That selected universe contains `{python} r.annotated_exact_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine's computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer's hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot's scored reference outputs come from the fixed engine, policyengine-us `{python} r.policyengine_us_version`. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. @tbl-reference-review summarizes the reviewed discrepancy classes and outcomes.
-The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged `{python} r.audit_flagged_case_count` references. Adjudication against primary sources and sandbox runs of the reference engine affirmed `{python} r.audit_flagged_count("affirmed")` of them, traced `{python} r.audit_flagged_count("unlisted_input")` to inputs the prompt never lists, replaced `{python} r.audit_flagged_count("regenerated")` with a regenerated reference, and confirmed `{python} r.audit_flagged_count("engine_defect")` as defects in policyengine-us `{python} r.previous_policyengine_us_version`, the version behind the references then. The judge's stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause's rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, `{python} r.upstream_fixed_root_cause_count` had been fixed in policyengine-us after the references were frozen (`{python} r.upstream_fix_prs_fmt`): capital gain distributions, New York's renter cap, the CalEITC's lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, `{python} r.regenerated_by_upstream_fix_count` in all, and policyengine-us `{python} r.policyengine_us_version` contains all `{python} r.upstream_fixed_root_cause_count` fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin's 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin's published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model: `{python} r.engine_defect_exclusion_count` outputs in `{python} r.excluded_output_households_by_reason["reference_engine_defect"]` households are recorded as engine-defect exclusions across `{python} r.engine_defect_root_cause_count` root causes, and `{python} r.engine_defect_unflagged_count` of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). They stay excluded until references built on a fixed engine replace them. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household's survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.
+The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged `{python} r.audit_flagged_case_count` references. Adjudication against primary sources and sandbox runs of the reference engine affirmed `{python} r.audit_flagged_count("affirmed")` of them, traced `{python} r.audit_flagged_count("unlisted_input")` to inputs the prompt never lists, replaced `{python} r.audit_flagged_count("regenerated")` with a regenerated reference, and confirmed `{python} r.audit_flagged_count("engine_defect")` as defects in policyengine-us `{python} r.september_upgrade.previous_engine_version`, the version behind the references then. The judge's stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause's rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, `{python} r.upstream_fixed_root_cause_count` had been fixed in policyengine-us after the references were frozen (`{python} r.upstream_fix_prs_fmt`): capital gain distributions, New York's renter cap, the CalEITC's lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, `{python} r.regenerated_by_upstream_fix_count` in all, and policyengine-us `{python} r.policyengine_us_version` contains all `{python} r.upstream_fixed_root_cause_count` fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin's 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin's published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model. The exclusion record holds `{python} r.engine_defect_exclusion_count` engine-defect exclusions in `{python} r.excluded_output_households_by_reason["reference_engine_defect"]` households across `{python} r.engine_defect_root_cause_count` root causes, and `{python} r.engine_defect_unflagged_count` of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). policyengine-us `{python} r.policyengine_us_version` still has the defect behind `{python} r.engine_defect_unfixed_count` of them, across `{python} r.engine_defect_unfixed_root_cause_count` root causes. An output returns to scoring when references built on an engine that fixes its defect replace it, unless it also moves under an input the prompt does not state or its reference rests on a further defect that engine has. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household's survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.
The same wave tested a failure mode no judge flagged. policyengine-us fills a parameter's values past its last encoded year by projecting it with a price index, so a 2026 reference can rest on a forecast rather than on the amount a government published. Every reference was recomputed with each projected 2026 value held at its last encoded value, and each projected amount behind a moved output was then sourced. Where the government published the 2026 amount before the freeze, the reference takes it: Minnesota's, Michigan's and Missouri's brackets, deductions and exemptions, and Maryland's withholding allowance. policyengine-us reads the allowance only in its estimate of Maryland income tax withheld, which fills the state income tax part of the federal state and local tax deduction; the one Maryland federal output that estimate moves is excluded (`reference_audit/2026-10-05`). Wisconsin had published its 2026 brackets and deductions too, but every Wisconsin output they move is excluded for an engine defect, so they enter only those outputs' corrected values. Where no 2026 amount published before the freeze was found, the reference takes the last amount published: California's indexed amounts, whose 2026 factor rests on June 2026 prices; the IRS optional sales tax tables, whose 2026 edition had not been found by September 22; Idaho's zero-rate threshold; Maryland's return deduction; and SNAP's FY2027 figures, published on 2026-08-21. Illinois's exemption and Minnesota's child credit projections matched the published amounts. These corrections change parameter values and no rule, so the affected references were regenerated with the same engine version rather than excluded: `{python} r.regenerated_by_convention_count` references, most of them SNAP. With the upstream fixes, the September 22 release regenerated `{python} r.regenerated_reference_count` references in `{python} r.regenerated_reference_household_count` households, `{python} r.regenerated_snap_reference_count` of them SNAP. The reference sidecar records each regenerated value with the fix that produced it, and the fixes, the recomputation of every reference under each, and the verification reports are committed with the snapshot (`reference_audit/2026-09-22`).
The SNAP convention as first built also corrected the engine's SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine's household size, eligibility and floored net income, the Axiom rules engine's encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey's state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income's cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect is not fixed upstream: the engine still grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, and the scored output it moves is excluded. That household's prompt calls its head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits; read the other way, the allowance would stand, and so would the reference the publication conventions and upstream SNAP fixes give ($3,576, which the output would carry if scored), so the output is excluded on either reading. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine's values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA's SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan's broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0.
-PolicyBench rebuilt the references on `{python} r.reference_rebuilt_date` with policyengine-us `{python} r.policyengine_us_version`, the newest release when it began sweeping the references that day (uploaded `{python} r.reference_engine_uploaded_utc` UTC), in place of `{python} r.previous_policyengine_us_version`. It computes each scored reference with `policyengine_us.Simulation`. The snapshot also records policyengine.py `{python} r.policyengine_version` for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside `{python} r.policyengine_us_version`. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA's FY2027 figures as published values. policyengine-us `{python} r.policyengine_us_version` counts Maryland county income tax in its state income tax. PolicyBench's state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP's work rules read, so the scenario builder also passes each prompt's stated usual weekly hours under that input's name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes `{python} r.engine_upgrade_scored_change_count` scored references (@tbl-reference-review). Two take law published before the freeze that policyengine-us `{python} r.previous_policyengine_us_version` did not encode: New Jersey's child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona's broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York's Empire State child credit phase-out. The investigation found, and PolicyBench excluded, `{python} r.engine_upgrade_new_exclusion_count` federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household's excluded SNAP output also flagged a scored output the move did not change: the head's Medicaid eligibility. The head's income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California's Working Disabled Program requires SSI's definition of disability, the unlisted input that already excludes the household's SNAP. policyengine-us `{python} r.policyengine_us_version` tests the general disability flag there and gives 1 under either reading; with the program's test reading SSI's definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another `{python} r.engine_upgrade_within_tolerance_count` references move by less than the dollar tolerance. PolicyBench re-reviewed the `{python} r.engine_upgrade_rechecked_count` excluded outputs whose values move on `{python} r.policyengine_us_version`; all stay excluded, each with the value it was decided on. policyengine-us `{python} r.publication_check_policyengine_us_version`, the newest release when PolicyBench checked PyPI on `{python} r.publication_check_pypi_read_date` at `{python} r.publication_check_pypi_read_utc` UTC, gives the same value as `{python} r.policyengine_us_version` for all `{python} r.total_outputs_per_model_fmt` outputs under the same conventions and adapter. The reference sidecar lists every change, and `reference_audit/2026-09-28` holds the recomputation of every output, the investigations and the reviews.
+PolicyBench rebuilt the references on `{python} r.september_upgrade.date` with policyengine-us `{python} r.september_upgrade.engine_version`, the newest release when it began sweeping the references that day (uploaded `{python} r.reference_engine_uploaded_utc` UTC), in place of `{python} r.september_upgrade.previous_engine_version`. It computes each scored reference with `policyengine_us.Simulation`. The snapshot also records policyengine.py `{python} r.policyengine_version` for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside `{python} r.september_upgrade.engine_version`. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA's FY2027 figures as published values. policyengine-us `{python} r.september_upgrade.engine_version` counts Maryland county income tax in its state income tax. PolicyBench's state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP's work rules read, so the scenario builder also passes each prompt's stated usual weekly hours under that input's name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes `{python} r.september_upgrade.scored_change_count` scored references (@tbl-reference-review). Two take law published before the freeze that policyengine-us `{python} r.september_upgrade.previous_engine_version` did not encode: New Jersey's child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona's broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York's Empire State child credit phase-out. The investigation found, and PolicyBench excluded, `{python} r.september_upgrade.new_exclusion_count` federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household's excluded SNAP output also flagged a scored output the move did not change: the head's Medicaid eligibility. The head's income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California's Working Disabled Program requires SSI's definition of disability, the unlisted input that already excludes the household's SNAP. policyengine-us `{python} r.september_upgrade.engine_version` tests the general disability flag there and gives 1 under either reading; with the program's test reading SSI's definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another `{python} r.september_upgrade.within_tolerance_count` references move by less than the dollar tolerance. PolicyBench re-reviewed the `{python} r.september_upgrade.rechecked_count` excluded outputs whose values move on `{python} r.september_upgrade.engine_version`; all stay excluded, each with the value it was decided on. policyengine-us `{python} r.publication_check_policyengine_us_version`, the newest release when PolicyBench checked PyPI on `{python} r.publication_check_pypi_read_date` at `{python} r.publication_check_pypi_read_utc` UTC, gives the same value as `{python} r.september_upgrade.engine_version` for all `{python} r.total_outputs_per_model_fmt` outputs under the same conventions and adapter. The reference sidecar lists every change, and `reference_audit/2026-09-28` holds the recomputation of every output, the investigations and the reviews.
-A review of release dashboard-data-20260930 on `{python} r.review_date` found `{python} r.review_sweep_count_word` more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (`reference_audit/2026-10-05`, `reference_audit/2026-10-05-medicare-part-b` and `reference_audit/2026-10-05-payroll`). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with `state_withheld_income_tax`, a formula estimate of withholding on the person's whole federal adjusted gross income rather than the household's state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of `{python} r.review_sweep_moved_count_word("salt_withholding")` households moves, `{python} r.review_sweep_newly_excluded_count_word("salt_withholding")` of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt's instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of `{python} r.review_sweep_moved_household_count_word("medicare_part_b")` household move: it itemizes, so its federal income tax moves, and Virginia's itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer's choice. The `{python} r.review_sweep_moved_count_word("payroll_optional_shares")` payroll references from those states count the cap, and the prompt does not state the employer's choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California's, Connecticut's, New Jersey's and Pennsylvania's employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us `{python} r.policyengine_us_version` with the publication conventions, under each reading, and each sweep's baseline reproduces all `{python} r.review_scored_before_count_fmt` references that release scored. Beyond the one-dollar tolerance, the sweeps move `{python} r.review_new_exclusion_count_word` scored outputs, which PolicyBench excluded, and `{python} r.review_already_excluded_moved_count_word` federal income tax outputs already excluded for other reasons; they move `{python} r.review_still_scored_moves_phrase` by any amount. The `{python} r.review_new_exclusion_count_word` records are computed on policyengine-us `{python} r.review_exclusion_engine_version`, whose values are the references. The `{python} r.excluded_output_count` excluded outputs now keep the values they were decided on (`{python} r.excluded_outputs_on_previous_engine_count` computed with policyengine-us `{python} r.previous_policyengine_us_version`, `{python} r.excluded_outputs_on_reference_engine_count` with `{python} r.policyengine_us_version`).
+A review of release dashboard-data-20260930 on `{python} r.review_date` found `{python} r.review_sweep_count_word` more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (`reference_audit/2026-10-05`, `reference_audit/2026-10-05-medicare-part-b` and `reference_audit/2026-10-05-payroll`). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with `state_withheld_income_tax`, a formula estimate of withholding on the person's whole federal adjusted gross income rather than the household's state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of `{python} r.review_sweep_moved_count_word("salt_withholding")` households moves, `{python} r.review_sweep_newly_excluded_count_word("salt_withholding")` of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt's instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of `{python} r.review_sweep_moved_household_count_word("medicare_part_b")` household move: it itemizes, so its federal income tax moves, and Virginia's itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer's choice. The `{python} r.review_sweep_moved_count_word("payroll_optional_shares")` payroll references from those states count the cap, and the prompt does not state the employer's choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California's, Connecticut's, New Jersey's and Pennsylvania's employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us `{python} r.review_exclusion_engine_version` with the publication conventions, under each reading, and each sweep's baseline reproduces all `{python} r.review_scored_before_count_fmt` references that release scored. Beyond the one-dollar tolerance, the sweeps move `{python} r.review_new_exclusion_count_word` scored outputs, which PolicyBench excluded, and `{python} r.review_already_excluded_moved_count_word` federal income tax outputs already excluded for other reasons; they move `{python} r.review_still_scored_moves_phrase` by any amount. The `{python} r.review_new_exclusion_count_word` records are computed on policyengine-us `{python} r.review_exclusion_engine_version`, whose values are the references.
+
+On `{python} r.ruling_date` PolicyBench ruled to exclude `{python} r.ruled_exclusion_count_word` more outputs. `{python} r.ruled_decision_count_word("d1022").capitalize()` came from a reference adversary (`reference_audit/2026-10-05-reference-adversary`). It flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give. A judge that cannot yet see how the engine derived the reference works each flagged output from primary law, and only then reconciles its answer with the derivation. The adversary flagged 61 outputs and ran 52, the other nine belonging to other audits, and it held 45 references. An independent check of its seven other verdicts confirmed four as engine defects: Arizona's 2026 standard deduction left unindexed (A.R.S. 43-1041(H)), Ohio's medical deduction counting health insurance premiums only for a Medicare-eligible filer with no employer-paid plan (R.C. 5747.01(A)(10)), Colorado's 2026 sales tax refund paid without the excess state revenues that C.R.S. 39-22-2003(2) requires, and New York's 2026 child and dependent care credit, which Tax Law 606(c-2) replaced. The check refuted two, the Medicaid eligibility of two children in a North Carolina household, whose references stand. The seventh is a question of definition. A Pennsylvania household's dependent earns $45,000 of wages and must file their own return, and the output definitions name no return, so the household's federal and state income tax can be read with or without that return; a scan for the same question found a Missouri household, and PolicyBench excluded the four outputs. The other `{python} r.ruled_decision_count_word("d994")` outputs are two Louisiana households' 2026 state income tax (`reference_audit/2026-10-05-louisiana`). Their references rest on a standard deduction that policyengine-us computed from published price indexes under Louisiana's formula; Louisiana published the return amount itself on 2026-09-28, after the freeze. All `{python} r.ruled_exclusion_count_word` records are computed on policyengine-us `{python} r.ruled_exclusion_engine_version`.
+
+On `{python} r.engine_upgrade_date` PolicyBench moved the references again, from policyengine-us `{python} r.previous_policyengine_us_version` to `{python} r.policyengine_us_version`, a release that fixes engine defects behind excluded outputs. With the same publication conventions and adapter, it recomputed every output (`reference_audit/2026-10-09-engine-upgrade`). `{python} r.engine_upgrade_timing_sentence` `{python} r.engine_upgrade_restored_split_sentence` `{python} r.engine_upgrade_restored_target_sentence` The reference sidecar records each target with the upstream fix it names (`regenerated_exclusions`). `{python} r.engine_upgrade_restored_sentence` The move changes `{python} r.engine_upgrade_scored_change_count_word` scored references beyond the dollar tolerance and `{python} r.engine_upgrade_within_tolerance_count` within it, and removes `{python} r.engine_upgrade_new_exclusion_count_word` outputs from scoring. `{python} r.engine_upgrade_scored_change_sentence` `{python} r.engine_upgrade_new_exclusion_sentence` PolicyBench re-reviewed the `{python} r.engine_upgrade_rechecked_count` excluded outputs whose values move on `{python} r.policyengine_us_version`; all stay excluded, each with the value it was decided on. The `{python} r.excluded_output_count` excluded outputs now keep the values they were decided on (`{python} r.excluded_outputs_by_engine_version_phrase`).
The audit tooling itself required hardening, which we report because it changes how much the annotations can be trusted. The derivation narratives displayed alongside reference values are generated from engine computation traces. After earlier model-written narratives were found to misattribute eligibility pathways, mechanical validators now enforce that each narrative states the engine's actual mechanism — for example, that a child ineligible for CHIP because they qualify for Medicaid is never described as failing a CHIP income test. The failure-audit judge reads these grounded narratives together with engine facts and must return a per-model diagnosis naming the specific rule or computation the model missed. It is barred from re-adjudicating the reference in prose: hedged or reference-adjudicating verdicts are mechanically rejected and re-judged, and genuine reference doubt is routed to a structured flag that triggers the manual adjudication described above.
-After freezing the snapshot and completing response-contract repairs, we annotated the `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule` and their scenario-output cases. @tbl-deviation-audit reports the final classifications for that legacy-threshold audit universe. It contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits. The `{python} r.unannotated_below_full_bounded_score_count_fmt` other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: `llm_error` (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or `parse_contract_failure` (a missing or unparsable answer, classified deterministically). Two further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. `prompt_ambiguity` marks `{python} r.unlisted_input_exclusion_count` outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine's Working Disabled Program test read SSI's definition, as the program does). `reference_engine_defect` marks the `{python} r.engine_defect_exclusion_count` outputs where that engine misapplies the law on stated facts. Together they remove `{python} r.excluded_output_phrase` in `{python} r.excluded_output_households_phrase`, and of the `{python} r.excluded_output_annotation_row_count_fmt` annotated rows on those outputs, `{python} f"{r.excluded_descriptive_row_count:,}"` carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP's October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on `{python} r.scored_outputs_per_model_fmt` of its `{python} r.total_outputs_per_model_fmt` requested outputs. The exclusion record (`reference_exclusions.json`) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records `{python} r.audit_adjudicated_case_phrase` of developer adjudication, kept beside the annotations with the judge's verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the `{python} r.audit_opus_judged_case_count_fmt` cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29 and September 30 additions joined or a reference revision changed (`{python} r.audit_opus55_judged_case_count_fmt` of `{python} r.audit_case_count_fmt` cases), its own row's cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (`meets_ssi_disability_criteria`), false for every person in the certified June 2026 microdata build; months of SSDI receipt (`months_receiving_social_security_disability`), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (`weekly_hours_worked_before_lsr`), which policyengine-us `{python} r.previous_policyengine_us_version` defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin's homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers' child, which decides whose income counts in that dependent's Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general `is disabled` fact and never a program-specific determination (@sec-disability). Every exclusion but the California Medicaid output was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect's sandbox fix, moved on policyengine-us `{python} r.previous_policyengine_us_version` or, for the state and local tax refund reading and the readings of the `{python} r.review_date` review, on `{python} r.policyengine_us_version`. On `{python} r.previous_policyengine_us_version`, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On `{python} r.policyengine_us_version`, PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves `{python} r.rerun_sweep_new_excluded_count_word` previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same `{python} r.policyengine_us_version` calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and `{python} r.rerun_sweep_scored_within_tolerance_count_word` scored outputs move by less. PolicyBench did not re-run the other sweeps on `{python} r.policyengine_us_version`. On `{python} r.policyengine_us_version`, the California Medicaid output stays at 1 under either reading of SSI's disability criterion, because the engine's Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household's excluded SNAP output, and excluded it because the program itself requires SSI's definition of disability.
+After freezing the snapshot and completing response-contract repairs, we annotated the `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule` and their scenario-output cases. @tbl-deviation-audit reports the final classifications for that legacy-threshold audit universe. It contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits. The `{python} r.unannotated_below_full_bounded_score_count_fmt` other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: `llm_error` (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or `parse_contract_failure` (a missing or unparsable answer, classified deterministically). Three further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. `prompt_ambiguity` marks `{python} r.unlisted_input_exclusion_count` outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine's Working Disabled Program test read SSI's definition, as the program does). `reference_engine_defect` marks the `{python} r.engine_defect_exclusion_count` outputs where that engine misapplies the law on stated facts. `{python} r.engine_defect_fixed_kept_sentence` `{python} r.engine_defect_further_defect_sentence` `reference_later_law` marks the `{python} r.later_law_exclusion_count_word` Louisiana outputs whose reference rests on an amount the state published after the freeze. Together they remove `{python} r.excluded_output_phrase` in `{python} r.excluded_output_households_phrase`, and of the `{python} r.excluded_output_annotation_row_count_fmt` annotated rows on those outputs, `{python} f"{r.excluded_descriptive_row_count:,}"` carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP's October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on `{python} r.scored_outputs_per_model_fmt` of its `{python} r.total_outputs_per_model_fmt` requested outputs. The exclusion record (`reference_exclusions.json`) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records `{python} r.audit_adjudicated_case_phrase` of developer adjudication, kept beside the annotations with the judge's verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the `{python} r.audit_opus_judged_case_count_fmt` cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29, September 30 and October 9 additions joined or a reference revision changed (`{python} r.audit_opus55_judged_case_count_fmt` of `{python} r.audit_case_count_fmt` cases), its own row's cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (`meets_ssi_disability_criteria`), false for every person in the certified June 2026 microdata build; months of SSDI receipt (`months_receiving_social_security_disability`), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (`weekly_hours_worked_before_lsr`), which policyengine-us `{python} r.september_upgrade.previous_engine_version` defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin's homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers' child, which decides whose income counts in that dependent's Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general `is disabled` fact and never a program-specific determination (@sec-disability). Every exclusion but the California Medicaid output and the `{python} r.ruling_date` records was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect's sandbox fix, moved on policyengine-us `{python} r.september_upgrade.previous_engine_version` or, for the state and local tax refund reading and the readings of the `{python} r.review_date` review, on `{python} r.september_upgrade.engine_version`; the outputs the `{python} r.engine_upgrade_date` upgrade excluded are ones policyengine-us `{python} r.policyengine_us_version` itself moved, onto an input no prompt states. The `{python} r.ruling_date` records are the exception by design. The reference adversary selects the outputs where models agree against the reference, and an independent check against primary law then decided each; the two Louisiana records came from a sweep of that state's 2026 standard deduction. On `{python} r.september_upgrade.previous_engine_version`, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On `{python} r.september_upgrade.engine_version`, PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves `{python} r.rerun_sweep_new_excluded_count_word` previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same `{python} r.september_upgrade.engine_version` calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and `{python} r.rerun_sweep_scored_within_tolerance_count_word` scored outputs move by less. PolicyBench did not re-run the other sweeps on `{python} r.september_upgrade.engine_version`. On `{python} r.september_upgrade.engine_version`, the California Medicaid output stays at 1 under either reading of SSI's disability criterion, because the engine's Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household's excluded SNAP output, and excluded it because the program itself requires SSI's definition of disability.
```{python}
#| label: tbl-reference-review
@@ -1259,9 +1264,9 @@ PolicyBench is not a substitute for a production tax-and-benefit calculator. Sev
- the `{python} r.n_households_fmt`-household manuscript snapshot should be treated as a preview until larger frozen runs are published
- the annotations cover the `{python} r.audit_annotated_row_count_fmt`-row legacy-threshold audit universe, including `{python} r.annotated_exact_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits, but not `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
- one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (disability, which the prompt states as one general fact where each program applies its own determination, and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address (@sec-disability); outputs whose reference moves under the other reading of either convention are excluded, while the mis-encoded scenario's cells score models against its encoded inputs
-- the reference engine carries known defects: `{python} r.engine_defect_exclusion_count` outputs, recorded as engine-defect exclusions, rest on `{python} r.engine_defect_root_cause_count` root causes that the audit found in policyengine-us `{python} r.previous_policyengine_us_version` and that `{python} r.policyengine_us_version` does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, the engine-upgrade review covered only the outputs the upgrade moved, and on `{python} r.policyengine_us_version` PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
+- the reference engine carries known defects: `{python} r.engine_defect_present_count` outputs, recorded as engine-defect exclusions, rest on `{python} r.engine_defect_present_root_cause_count` root causes that the audits found in policyengine-us `{python} r.september_upgrade.previous_engine_version` or later releases and that `{python} r.policyengine_us_version` does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, each engine-upgrade review covered only the outputs its upgrade moved, and on `{python} r.september_upgrade.engine_version` PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
- the `{python} r.review_date` review of release dashboard-data-20260930 found `{python} r.review_new_exclusion_count_word` scored references that rested on inputs the prompt never states and that no judge had flagged, so other such references can remain scored: the exclusions cover the inputs the audits found
-- the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP's FY2026 figures rather than USDA's FY2027 figures of 2026-08-21; California's 2025 indexed amounts; the IRS's 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey's 2026 child tax credit schedule and Arizona's SNAP categorical eligibility limit only after `{python} r.previous_policyengine_us_version`
+- the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP's FY2026 figures rather than USDA's FY2027 figures of 2026-08-21; California's 2025 indexed amounts; the IRS's 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey's 2026 child tax credit schedule and Arizona's SNAP categorical eligibility limit only after `{python} r.september_upgrade.previous_engine_version`
- benchmark success should not be interpreted as policy-advice readiness
The current paper is therefore an evaluation of model performance under a specific structured-output benchmark, not a general certification of tax or benefit competence.
diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json
index 7a1aa643..35abfa10 100644
--- a/paper/snapshot/20260501/manifest.json
+++ b/paper/snapshot/20260501/manifest.json
@@ -2,55 +2,61 @@
"audit_annotation_artifacts": {
"developer_adjudications": {
"by_judge_verdict": {
- "llm_error": 67,
+ "llm_error": 59,
+ "prompt_ambiguity": 6,
"reference_data_issue_fixed": 1,
- "reference_engine_defect": 3,
- "reference_model_issue_fixed": 6
+ "reference_engine_defect": 1,
+ "reference_later_law": 1,
+ "reference_model_issue_fixed": 2
},
"by_reference_verdict": {
"affirmed": 5,
- "engine_defect": 28,
- "regenerated": 6,
- "unlisted_input": 28
+ "engine_defect": 14,
+ "later_law": 2,
+ "regenerated": 5,
+ "unlisted_input": 34
},
- "cases": 77,
+ "cases": 70,
"file": "us_adjudications.json",
"judge_flagged_by_reference_verdict": {
"affirmed": 5,
- "engine_defect": 20,
- "regenerated": 6,
+ "engine_defect": 11,
+ "later_law": 1,
+ "regenerated": 5,
"unlisted_input": 8
},
"note": "Judge verdicts outside the final classes (llm_error, parse_contract_failure) are resolved by a recorded developer adjudication that keeps the judge's verdict and reasoning beside the adjudicated class; applied to the bundle before export so the published payload and the frozen annotations agree. Every case the judge flagged reference-suspect carries a reference verdict (affirmed, regenerated, engine_defect, unlisted_input, later_law) that clears the flag; judge_reference_suspect records which entries the judge flagged."
},
"files": {
- "us_adjudications.json": "38145030badfc6aa3fad6289ccc6aa940e523d3c4a8dd70d9d4dbe104644ecf1",
- "us_audit_row_annotations.csv": "155128bb8951a22a99eb9f632395fc1ea344d890ab6f4c7e467747d1599b700c",
- "us_case_notes.csv": "660d0da57ffc08d1a99f35efdbfdc62373ce1055b58c3bc933c9f2d082106ebf",
- "us_case_reference_explanations.csv": "18787448808ec4fdbfbe092d385e6ad7329d4d7d919658f1bcc743f63b53afbe"
+ "us_adjudications.json": "939261c302336fcb8b60c1baec00039aff83a394a96fc5cde9dab979a9160530",
+ "us_audit_row_annotations.csv": "3f0ce2d273fd7c4e0a621a0a99927cf80b1e1b800f7da1d586c1fa58982985fd",
+ "us_case_notes.csv": "75f5b509959dbf8ef678ba6361f3f38c78ad7d11976583dc18e5925c30c0ec28",
+ "us_case_reference_explanations.csv": "7ef73f3ff003819eee9dd8bf5ba7b961bf27cfc863455be1f790a36f43498f5f"
},
"judge_provenance": {
"by_judge": {
"claude-opus-5": {
- "cases": 116,
+ "cases": 94,
"judged_on_utc": [
"2026-09-05"
],
"runner": "Claude Code CLI (scripts/run_audit_claude.sh)"
},
"claude-opus-5-5": {
- "cases": 260,
+ "cases": 314,
"judged_on_utc": [
- "2026-09-22",
"2026-09-23",
"2026-09-24",
"2026-09-29",
- "2026-09-30"
+ "2026-09-30",
+ "2026-10-08",
+ "2026-10-09",
+ "2026-10-10"
],
"runner": "Claude Code CLI (scripts/run_audit_claude.sh) and Claude Code Workflow subagents (results/local/adds202609/judge_stages.py)"
},
"gpt-5.6-sol": {
- "cases": 298,
+ "cases": 269,
"judged_on_utc": [
"2026-08-18",
"2026-08-22",
@@ -59,23 +65,23 @@
"runner": "Codex CLI (scripts/run_audit_codex.sh)"
}
},
- "cases_judged": 674,
+ "cases_judged": 677,
"note": "Judge model per case: the verdict.meta.json sidecar bound to the case's verdict by sha256 (either runner), else, for a case with no sidecar, the codex.log model header when the log was written alongside the verdict (Codex runner before it wrote sidecars). Verdicts classify misses after scoring and change no score. Every judge model is also a board row."
},
- "note": "Model-assisted, developer-adjudicated row and case audit annotations for the 9,971 prediction rows selected because their legacy threshold score was below 1. This audit universe is not identical to the exact-match or bounded-score miss sets. The annotations were produced under the decisive-diagnosis contract (per-model diagnoses grounded in engine facts; hedged verdicts mechanically rejected and re-judged). Row-level failure_source values are llm_error for substantive misses and parse_contract_failure for missing or unparseable answers; zero rows are reference-suspect. Case notes are stored as us_case_notes.csv with case_failure_sources / case_failure_subtypes columns. Reference narratives the judge and dashboard display are frozen as us_case_reference_explanations.csv. Judge provenance per case is tallied under judge_provenance.",
+ "note": "Model-assisted, developer-adjudicated row and case audit annotations for the 10,213 prediction rows selected because their legacy threshold score was below 1. This audit universe is not identical to the exact-match or bounded-score miss sets. The annotations were produced under the decisive-diagnosis contract (per-model diagnoses grounded in engine facts; hedged verdicts mechanically rejected and re-judged). Row-level failure_source values are llm_error for substantive misses and parse_contract_failure for missing or unparseable answers; zero rows are reference-suspect. Case notes are stored as us_case_notes.csv with case_failure_sources / case_failure_subtypes columns. Reference narratives the judge and dashboard display are frozen as us_case_reference_explanations.csv. Judge provenance per case is tallied under judge_provenance.",
"path": "annotations/us_full_run_20260612_policyengine_4_16_1_populace"
},
"committed_snapshot_artifacts": {
- "model_serving_config.json": "019e0b9f2987102b49f82b238ca30568cf403c6931444742f4a975a471d1da15",
- "us_impact_summary_by_model.csv": "bc9162a6a1a5f7e6c40bba31935d172b4338ad84a4d7a92e95c18fd84c03864a",
- "us_reference_outputs.csv": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466",
+ "model_serving_config.json": "e60915eff7d7900ba671df7ddb51ed4b5aa39489ea26fbf5a9beb41e45bfeae7",
+ "us_impact_summary_by_model.csv": "f603f7e5b9b3b7dc0cf71ab264ca903c438619f806a65a69749af6387f58b841",
+ "us_reference_outputs.csv": "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371",
"us_scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a"
},
- "description": "The 2026-09-30 scored manuscript snapshot reports the household-impact-weighted exact-match rate as its headline metric.",
+ "description": "The 2026-10-10 scored manuscript snapshot reports the household-impact-weighted exact-match rate as its headline metric.",
"files": [
{
"path": "runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz",
- "sha256": "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d"
+ "sha256": "535a51db14f13284b7054f2980b3663c3bc4317037eb3a0cc65e392096df5a90"
}
],
"household_dataset": {
@@ -87,14 +93,14 @@
},
"live_dashboard_artifact": {
"asset": "dashboard-data.json",
- "bytes": 127225036,
+ "bytes": 128756688,
"derivation": "At freeze time the live artifact equals the frozen published_dashboard_artifact: the combined export of the source-run data.json listed under source_run_artifacts. Annotation-class republishes may advance this entry ahead of the frozen pin without changing any score.",
- "sha256": "aa34e5c9ea926848dc7af460a98f17021bc42dae54a8015c9988a85e79a5d462",
- "tag": "dashboard-data-20261006",
- "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261006/dashboard-data.json"
+ "sha256": "f834478e65191f8426ef479506f9f6fe32c2e7107f30fdbf4e7ba0e6acba6410",
+ "tag": "dashboard-data-20261010",
+ "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261010/dashboard-data.json"
},
"live_dashboard_note": "The live dashboard payload is a published release asset; the committed pointer app/src/data.artifact.json must reference the artifact pinned under live_dashboard_artifact. The separate published_dashboard_artifact freezes the combined export of the source run data.json files listed under source_run_artifacts. A later publication may advance the live entry without changing the frozen pin.",
- "model_response_date": "2026-06-12 to 2026-09-29",
+ "model_response_date": "2026-06-12 to 2026-10-08",
"policy_period": {
"us": "tax year 2026"
},
@@ -105,29 +111,26 @@
},
"published_dashboard_artifact": {
"asset": "dashboard-data.json",
- "bytes": 127225036,
- "sha256": "aa34e5c9ea926848dc7af460a98f17021bc42dae54a8015c9988a85e79a5d462",
- "tag": "dashboard-data-20261006",
- "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261006/dashboard-data.json"
+ "bytes": 128756688,
+ "sha256": "f834478e65191f8426ef479506f9f6fe32c2e7107f30fdbf4e7ba0e6acba6410",
+ "tag": "dashboard-data-20261010",
+ "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20261010/dashboard-data.json"
},
"reference_exclusions": {
"by_engine_defect_root_cause": {
- "r01_ira_compensation": 3,
- "r02_ira_219g": 7,
- "r02_ira_219g+r03_estate_income": 2,
- "r02_ira_219g+r11_ca_itemized_conformity": 2,
- "r03_estate_income": 2,
- "r05_nj_worker_ui": 1,
+ "r02_ira_219g": 3,
+ "r02_ira_219g+r11_ca_itemized_conformity": 1,
"r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions": 1,
"r07_idaho_health_premiums": 2,
"r08_eitc_earned_income_deferrals": 4,
- "r11_ca_itemized_conformity": 1,
"r22_ma_part_a_loss_offset": 1,
"r30_snap_heat_and_eat_sua": 1,
"r32_wi_capital_gain_distributions": 1
},
"by_unlisted_input": {
"Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either": 1,
+ "county of residence (Indiana county income tax)": 2,
+ "la_2026_standard_deduction_unpublished": 2,
"meets_ssi_disability_criteria": 7,
"months_receiving_social_security_disability": 5,
"source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits": 1,
@@ -139,38 +142,39 @@
"weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies": 1,
"whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say": 1,
"whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax": 4,
+ "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own": 4,
"whether the listed home mortgage interest is on the home the SNAP household occupies": 1,
"whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)": 3,
"who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable": 4
},
"file": "reference_exclusions.json",
"note": "Outputs are removed from scoring for every model, symmetrically, when the reference depends on an input or definition the prompt never states (a careful reader could take the stated facts either way), or when the engine that produced the reference misapplies the law on facts the prompt states. Their rows stay in the payload with scored=false. An unlisted-input entry records the alternative reading and the reference under both readings; an engine-defect entry records the root cause, the law, the upstream issue, and the corrected value computed with the same engine version and a sandbox fix.",
- "outputs": 64,
- "scored_outputs_per_model": 1920
+ "outputs": 58,
+ "scored_outputs_per_model": 1926
},
"reference_output_refresh": {
- "date": "2026-09-29",
+ "date": "2026-10-10",
"generated_at_utc": "2026-07-03T04:21:31.815629+00:00",
"policyengine_us_data_artifact_sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84",
"policyengine_us_data_build_id": "populace-us-2024-spm-20260915",
"policyengine_us_dataset": "populace_us_2024",
"policyengine_us_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915",
- "policyengine_us_version": "2.15.17",
+ "policyengine_us_version": "2.38.6",
"policyengine_version": "6.1.2",
- "reference_csv_sha256": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466",
- "regenerated_at_utc": "2026-09-29T15:04:45.724738+00:00",
+ "reference_csv_sha256": "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371",
+ "regenerated_at_utc": "2026-10-10T06:39:57.084656+00:00",
"row_count": 1984,
- "snapshot_date": "2026-09-30"
+ "snapshot_date": "2026-10-10"
},
"rendered_paper_artifacts": {
"pdf": {
"path": "app/public/paper/policybench.pdf",
- "sha256": "2cffc69cffdee871b9ee16c257f539a13669379492252d2869466f8192770ff4"
+ "sha256": "1607c71abaf35fae6f6d51576450c91747f8fb158485fc6ea64431e4c4c683c3"
},
"web": {
"files": {
- "figures/positive_zero_scatter.png": "ffa8ccdf1648e1b897744c6988e6c398898756823a2d07b12f90075f7c5ad656",
- "index.html": "c6dfba0c5718cb3cca026e1b2b61dfd4ed71daca5e4cad33480ed6b207f48e2c",
+ "figures/positive_zero_scatter.png": "f13ca00d6a6a2232ee658733507d6e0d39dfc8f639b74ec4b3f016c85acc7fb6",
+ "index.html": "31cd4141373acbcd2d474841f9e4dbf488df8dd33e62b70699a5ffe47bd42551",
"pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8",
"policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9",
"site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9",
@@ -192,11 +196,11 @@
},
"reproducibility_notes": [
"The top-level scenario, reference-output, and impact-summary CSVs are byte-identical to the corresponding compact source-run artifacts copied under paper/snapshot/20260501/runs/.",
- "Model responses were collected in waves between June 12 and September 29, 2026, as models were added to the board; each model's full 100-household run is a single consistent wave. PolicyBench computes each scored reference output with policyengine_us.Simulation from policyengine-us 2.15.17, using the household's own listed inputs, the nine publication conventions, the Maryland output-scope adapter and the scenario builder's stated-hours alias, as the reference sidecar's engine_upgrade revision pins them (fix_modules, builder); policyengine.py 6.1.2 is recorded for provenance only. The households were drawn from the certified PolicyEngine US populace dataset (populace-us-2024-5da5a95-20260611, populace_us_2024), as scenarios.csv.meta.json records (household_dataset); the reference_output_refresh dataset fields name the reference runtime's default dataset, which reference computation does not read.",
+ "Model responses were collected in waves between June 12 and October 8, 2026, as models were added to the board; each model's full 100-household run is a single consistent wave. PolicyBench computes each scored reference output with policyengine_us.Simulation from policyengine-us 2.38.6, using the household's own listed inputs, the nine publication conventions, the Maryland output-scope adapter and the scenario builder's stated-hours alias, as the reference sidecar's engine_upgrade revision pins them (fix_modules, builder); policyengine.py 6.1.2 is recorded for provenance only. The households were drawn from the certified PolicyEngine US populace dataset (populace-us-2024-5da5a95-20260611, populace_us_2024), as scenarios.csv.meta.json records (household_dataset); the reference_output_refresh dataset fields name the reference runtime's default dataset, which reference computation does not read.",
"Canonical prediction files include parser recovery. Later waves ran under the resumable supervised runner, which retries failed or timed-out scenarios in bounded rounds; every model's canonical file covers all 100 households.",
"Raw provider responses are retained in the compressed source-run predictions.csv.gz file wherever the transport exposed them; rows served through the Anthropic batch adapter (Claude Fable 5) and 64 Kimi K3 parse failures carry no raw payload. The separate LiteLLM cache remains local-only because it is a generated request cache, not the canonical snapshot artifact.",
"The frozen scenarios.csv source_dataset column carries a stale enhanced_cps_2024 label from the pre-#77 scenario generator; the run metadata (scenarios.csv.meta.json) records the populace_us_2024 build actually loaded.",
- "Model APIs and upstream model aliases may change after the recorded 2026-06-12 to 2026-09-29 response window, so exact reruns can diverge even with the committed household inputs, reference outputs, parsed dashboard export, and analysis summaries."
+ "Model APIs and upstream model aliases may change after the recorded 2026-06-12 to 2026-10-08 response window, so exact reruns can diverge even with the committed household inputs, reference outputs, parsed dashboard export, and analysis summaries."
],
"response_retry_artifacts": {
"files": {},
@@ -213,27 +217,27 @@
"households": {
"us": 100
},
- "models": 46,
+ "models": 47,
"output_groups": {
"us": 18
}
},
- "snapshot_date": "2026-09-30",
+ "snapshot_date": "2026-10-10",
"source_run_artifacts": {
"note": "Compact copies of run outputs used to verify this snapshot. The run data.json retains parsed scenario predictions, explanations, summaries, heatmaps, and PolicyEngine runtime metadata used by the dashboard. predictions.csv.gz is a deterministic gzip of the run's raw provider responses.",
"us_full_run_20260612_policyengine_4_16_1_populace": {
"files": {
- "analysis/impact_summary_by_model.csv": "bc9162a6a1a5f7e6c40bba31935d172b4338ad84a4d7a92e95c18fd84c03864a",
- "analysis/metrics.csv": "8d6dbac816ccccb0741952fb9d7e249381adbdcaf23b4cafb72652021626a8bb",
- "analysis/report.md": "afc35eae2027e13ab18b45d6b436648204d4b7b36697d7e0ddf1d2c2fd13d258",
- "analysis/summary_by_model.csv": "1b7640f3c0c7e7376c1bed0567dcc034866822a06e998658f4595ef74e5acabf",
- "analysis/summary_by_variable.csv": "39b9c2636ee59456905d65a8a43d7b7c511fa584875941f6f2dbcad54085ff46",
- "analysis/usage_summary.csv": "6d2f6338af5eb9db19dbfb0f1ba37363387b3280b85ae64feaa8dba2e2ccf9db",
- "data.json.gz": "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d",
- "predictions.csv.gz": "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12",
- "reference_exclusions.json": "92741dfd047298eacb79a0dadeb7e4965e3c6f986beab0341b12ff8f5b023815",
- "reference_outputs.csv": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466",
- "reference_outputs.csv.meta.json": "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b",
+ "analysis/impact_summary_by_model.csv": "f603f7e5b9b3b7dc0cf71ab264ca903c438619f806a65a69749af6387f58b841",
+ "analysis/metrics.csv": "938e10b7f7869ab508509f8af1b6c541d1817684105d280811ba8f715c735b3d",
+ "analysis/report.md": "e2aeb148b211560d530f3c778682b11e2305ea910a2a8a74a15f5edb1bdcc5b8",
+ "analysis/summary_by_model.csv": "d53214d3cad44584c4edd3f05b778fe4bac5227d06c3bd758a421e6c34be580f",
+ "analysis/summary_by_variable.csv": "09c5e5ecfe1eb66f58a3b2721837f348141b04e577143ad362d83a250e89c8fb",
+ "analysis/usage_summary.csv": "5ba7c01143f50881ddb828a65dbdbf1011f5480dc4acbd66b8178895593b49fb",
+ "data.json.gz": "535a51db14f13284b7054f2980b3663c3bc4317037eb3a0cc65e392096df5a90",
+ "predictions.csv.gz": "0183ab75931b354d6b177b147bbf57fbb79d9d0ac5b98a0304fa32eb329007ae",
+ "reference_exclusions.json": "089e01e1f699aa19f2171e1ac5b6dc2ce775f9898c94420e6a0b06ff5f773626",
+ "reference_outputs.csv": "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371",
+ "reference_outputs.csv.meta.json": "d534cfeacf7f8a8b7f6fca477adbb02c5bb52b226778d63cc10fd117ec70c54b",
"scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a",
"scenarios.csv.meta.json": "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb"
},
diff --git a/paper/snapshot/20260501/model_serving_config.json b/paper/snapshot/20260501/model_serving_config.json
index 99dcbe8c..eade7173 100644
--- a/paper/snapshot/20260501/model_serving_config.json
+++ b/paper/snapshot/20260501/model_serving_config.json
@@ -13,7 +13,7 @@
},
"evidence_summary": {
"registry": 29,
- "run_state": 17
+ "run_state": 18
},
"models": {
"claude-fable-5": {
@@ -108,6 +108,63 @@
"shared_completion_budget_tokens": null,
"tool_choice": "forced"
},
+ "claude-haiku-5.5": {
+ "answer_contract": "tool",
+ "evidence": {
+ "compared_columns": [
+ "scenario_id",
+ "variable",
+ "prediction",
+ "explanation",
+ "raw_response",
+ "provider_resolved_model",
+ "prompt_tokens",
+ "completion_tokens",
+ "error"
+ ],
+ "fields": [
+ "answer_contract",
+ "chunk_size",
+ "completion_budget_ceiling",
+ "fingerprint_version",
+ "initial_completion_budget_tokens",
+ "max_repair_rounds",
+ "model_id",
+ "prompt_contract_version",
+ "request_timeout_seconds",
+ "thinking",
+ "tool_choice_mode"
+ ],
+ "frozen_rows_sha256": "4a7373791a5f9df6e4c775b8aade37ecd58df87bd8b99f238889f1afe744621c",
+ "kind": "run_state",
+ "rows_match": true,
+ "run": "haiku55-runs/haiku55",
+ "source_predictions_sha256": "27a5e48f44ad875f6d0901571b91c06c67045a7a6841f95a5ab56ecf0cfc8344",
+ "source_rows_sha256": "4a7373791a5f9df6e4c775b8aade37ecd58df87bd8b99f238889f1afe744621c",
+ "treatment_fingerprint": {
+ "answer_contract": "tool",
+ "chunk_size": null,
+ "completion_budget_ceiling": 128000,
+ "fingerprint_version": 3,
+ "initial_completion_budget_tokens": 16384,
+ "max_repair_rounds": 2,
+ "model_id": "claude-haiku-5-5",
+ "prompt_contract_version": "2026-08-09-v2-scoring-contract",
+ "request_timeout_seconds": 300,
+ "thinking": {
+ "mode": "provider_default"
+ },
+ "tool_choice_mode": "forced"
+ }
+ },
+ "provider_id": "claude-haiku-5-5",
+ "reasoning_setup": "provider default; 16,384-token shared budget",
+ "registry_derived": [],
+ "request_shape": "whole scenario",
+ "request_timeout_seconds": 300,
+ "shared_completion_budget_tokens": 16384,
+ "tool_choice": "forced"
+ },
"claude-opus-4.7": {
"answer_contract": "tool",
"evidence": {
@@ -1528,7 +1585,7 @@
"tool_choice": null
}
},
- "registry_commit": "cddd01812ad6d6311502ef997f93634f70c28fda",
+ "registry_commit": "0562d77a9d263c14bb9c9fa93ba3c765fcff95a9",
"sources": [
"policybench.config.MODELS",
"policybench.model_cards",
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv
index d7a013c2..303eed76 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv
@@ -1,47 +1,48 @@
model,mean_impact_score,mean_household_score,mean_household_coverage,households,total_variables,parsed_variables,floor_share
-gpt-6-astra,0.9683122444817193,0.9784738250614189,1.0,100,1920,1920,0.3
-gpt-6-sol,0.9667129921990232,0.9825416199504653,1.0,100,1920,1920,0.3
-gpt-6-luna,0.9664167571502716,0.977394792016256,1.0,100,1920,1920,0.3
-claude-opus-5.5,0.962246515529886,0.982664785613371,1.0,100,1920,1920,0.3
-gpt-5.6-sol,0.9599111614346824,0.9765950579493266,1.0,100,1920,1920,0.3
-gpt-6.1-sol,0.9594811057246667,0.975488445843518,1.0,100,1920,1920,0.3
-claude-sonnet-5.5,0.9593949353595599,0.9733090827566719,1.0,100,1920,1920,0.3
-claude-fable-5.1,0.9569037993263287,0.9780947033412107,1.0,100,1920,1920,0.3
-grok-4.7,0.9544396463560192,0.9761065254261345,1.0,100,1920,1920,0.3
-inkling,0.9537243518627281,0.9708950810157603,1.0,100,1920,1920,0.3
-gpt-5.5,0.9463990239340224,0.9662756610360458,1.0,100,1920,1920,0.3
-deepseek-v4.1-flash,0.9449297542963264,0.9677712793442468,1.0,100,1920,1920,0.3
-gpt-5.6-terra,0.9428227924598422,0.9653030686475066,1.0,100,1920,1920,0.3
-gemini-3.8-flash,0.9413625215070209,0.9757756765309048,1.0,100,1920,1920,0.3
-grok-4.6,0.9370631371759138,0.9709023505552119,1.0,100,1920,1920,0.3
-gemini-3.6-flash,0.9353602488202646,0.9710758545154309,1.0,100,1920,1920,0.3
-ox-alpha,0.9349263306800009,0.9645184800680184,1.0,100,1920,1920,0.3
-kimi-k3,0.9321859170038987,0.947074103380015,0.97,100,1920,1862,0.3
-grok-4.5,0.9265179738707723,0.9637656553313277,1.0,100,1920,1920,0.3
-deepseek-v4-pro-0813,0.92592710415014,0.9666373591127111,1.0,100,1920,1920,0.3
-gpt-5.6-luna,0.9214893430494332,0.9595527347680398,1.0,100,1920,1920,0.3
-gemini-3-flash-preview,0.9194570538721496,0.9616863739278441,1.0,100,1920,1920,0.3
-deepseek-v4-flash-0731,0.9189428784905118,0.9549362454617714,1.0,100,1920,1920,0.3
-claude-opus-4.7,0.9178204428076171,0.9341214404196417,1.0,100,1920,1920,0.3
-claude-sonnet-4.6,0.9162129440743614,0.9452099117398467,1.0,100,1920,1920,0.3
-claude-fable-5,0.9137887992654609,0.9541658659499054,1.0,100,1920,1920,0.3
-claude-opus-5,0.9117042199103389,0.9569245656983695,1.0,100,1920,1920,0.3
-gemini-3.7-flash,0.9080053412243946,0.9640672785413511,1.0,100,1920,1920,0.3
-grok-build-0.1,0.907229168647409,0.9495641986927762,1.0,100,1920,1920,0.3
-gemini-3.1-pro-preview,0.9015422572930285,0.96182457021108,1.0,100,1920,1920,0.3
-deepseek-v4-pro,0.894759815215543,0.9531689498921042,1.0,100,1920,1920,0.3
-gemini-3.5-flash,0.8945050366123243,0.9571534285510584,1.0,100,1920,1920,0.3
-glm-5.3,0.8897481095502663,0.9276676213475469,0.97,100,1920,1849,0.3
-claude-opus-4.8,0.8826928093313342,0.9195533132045565,1.0,100,1920,1920,0.3
-claude-sonnet-5,0.8666788054918853,0.9162508180749187,1.0,100,1920,1920,0.3
-gemini-3.1-flash-lite-preview,0.8635748475506705,0.940345176887288,1.0,100,1920,1920,0.3
-qwen-3.7-max,0.8582890324943854,0.9260164229773913,1.0,100,1920,1920,0.3
-grok-4.3,0.8557494611738747,0.9440812980726867,1.0,100,1920,1920,0.3
-glm-5.2,0.8465447008531164,0.8995325933970066,0.9433301746876838,100,1920,1788,0.3
-qwen3.8-max,0.822354297921981,0.9071986559772072,1.0,100,1920,1920,0.3
-gemini-3.5-flash-lite,0.8157005424440896,0.9412320459816107,1.0,100,1920,1920,0.3
-claude-haiku-4.5,0.7972978664052773,0.9093026578365233,1.0,100,1920,1920,0.3
-gpt-5.4-mini,0.7646496681088202,0.9044299489482875,1.0,100,1920,1920,0.3
-minimax-m3,0.7486300831203856,0.9185494731869339,1.0,100,1920,1920,0.3
-gpt-5.4-nano,0.6976458164890331,0.8943748089883199,1.0,100,1920,1920,0.3
-kimi-k2.6,0.5929836623884213,0.7777325674444523,0.796733767182861,100,1920,1537,0.3
+gpt-6-sol,0.9683890379175923,0.9835781550408013,1.0,100,1926,1926,0.3
+gpt-6-astra,0.9656360677841757,0.9776432719593253,1.0,100,1926,1926,0.3
+gpt-6-luna,0.9655856177136641,0.977552888689478,1.0,100,1926,1926,0.3
+claude-opus-5.5,0.9645893550945023,0.983719384804741,1.0,100,1926,1926,0.3
+gpt-5.6-sol,0.9603411979737442,0.9772470545835048,1.0,100,1926,1926,0.3
+claude-fable-5.1,0.9596842169634039,0.9782975576452126,1.0,100,1926,1926,0.3
+gpt-6.1-sol,0.9577997782276715,0.9755996956654386,1.0,100,1926,1926,0.3
+claude-sonnet-5.5,0.9557169975683331,0.9732048971867467,1.0,100,1926,1926,0.3
+inkling,0.9553730764952827,0.9718310312144887,1.0,100,1926,1926,0.3
+grok-4.7,0.9530593671514012,0.9769017229703395,1.0,100,1926,1926,0.3
+gemini-3.8-flash,0.9462699389421277,0.977767902256248,1.0,100,1926,1926,0.3
+gpt-5.5,0.9453055193873112,0.9667817665830087,1.0,100,1926,1926,0.3
+deepseek-v4.1-flash,0.9436620453469386,0.9685038058811106,1.0,100,1926,1926,0.3
+gpt-5.6-terra,0.9412373765543233,0.9658277780925907,1.0,100,1926,1926,0.3
+ox-alpha,0.9384675548301299,0.9658066001187361,1.0,100,1926,1926,0.3
+kimi-k3,0.9369093783380158,0.9485146940042462,0.97,100,1926,1866,0.3
+gemini-3.6-flash,0.9351220326362744,0.9716546088284257,1.0,100,1926,1926,0.3
+grok-4.6,0.9338430747098238,0.9716311365558624,1.0,100,1926,1926,0.3
+grok-4.5,0.9247602994174631,0.9648720629656492,1.0,100,1926,1926,0.3
+deepseek-v4-pro-0813,0.9212125402654001,0.9671921672284504,1.0,100,1926,1926,0.3
+gpt-5.6-luna,0.9203959222079253,0.9600106363757288,1.0,100,1926,1926,0.3
+deepseek-v4-flash-0731,0.9173624769435257,0.9550956395701116,1.0,100,1926,1926,0.3
+gemini-3-flash-preview,0.9161481688270676,0.9621720429043401,1.0,100,1926,1926,0.3
+claude-opus-4.7,0.9117276162674928,0.9329098370223292,1.0,100,1926,1926,0.3
+claude-sonnet-4.6,0.9104613797385994,0.9449727716388616,1.0,100,1926,1926,0.3
+gemini-3.7-flash,0.907486043649933,0.9647129992653505,1.0,100,1926,1926,0.3
+grok-build-0.1,0.905886694331183,0.9499599365076756,1.0,100,1926,1926,0.3
+claude-fable-5,0.9054051934217692,0.9537747319332733,1.0,100,1926,1926,0.3
+claude-opus-5,0.9052680685413241,0.956209808032048,1.0,100,1926,1926,0.3
+gemini-3.1-pro-preview,0.9002961530624367,0.9625050398352599,1.0,100,1926,1926,0.3
+gemini-3.5-flash,0.8943613983201546,0.9582086228255206,1.0,100,1926,1926,0.3
+deepseek-v4-pro,0.8891412957260344,0.9530685519112764,1.0,100,1926,1926,0.3
+glm-5.3,0.8863036650145125,0.9270981657064452,0.97,100,1926,1853,0.3
+claude-opus-4.8,0.8763670208984979,0.9193961589624415,1.0,100,1926,1926,0.3
+gemini-3.1-flash-lite-preview,0.8549265116548262,0.9400571073735258,1.0,100,1926,1926,0.3
+claude-sonnet-5,0.8546597554533697,0.9145775517501825,1.0,100,1926,1926,0.3
+grok-4.3,0.8444550761064535,0.9431852757043429,1.0,100,1926,1926,0.3
+glm-5.2,0.8428927526586356,0.898248786758015,0.942134973522516,100,1926,1790,0.3
+qwen-3.7-max,0.8413035116697489,0.9245943379074073,1.0,100,1926,1926,0.3
+qwen3.8-max,0.8145343660950313,0.9054694717193006,1.0,100,1926,1926,0.3
+gemini-3.5-flash-lite,0.8048661218683532,0.9403942740096725,1.0,100,1926,1926,0.3
+claude-haiku-4.5,0.7820055915532537,0.9079458048084028,1.0,100,1926,1926,0.3
+claude-haiku-5.5,0.7508491615855171,0.9318432564796563,1.0,100,1926,1926,0.3
+gpt-5.4-mini,0.7438501974068404,0.901965060179966,1.0,100,1926,1926,0.3
+minimax-m3,0.7363778997274973,0.9163087437899782,1.0,100,1926,1926,0.3
+gpt-5.4-nano,0.6816490778744461,0.8929229972630024,1.0,100,1926,1926,0.3
+kimi-k2.6,0.5822018394629117,0.7752609667992933,0.7938565674923747,100,1926,1536,0.3
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv
index f2c42fa4..14a03e7e 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv
@@ -1,9 +1,9 @@
model,variable,n,n_parsed,coverage,mae,mape,accuracy,exact,within_1pct,within_5pct,within_10pct,threshold_score,score
-claude-fable-5,federal_income_tax_before_refundable_credits,79,79,1.0,563.4911491297469,0.5140834591717652,,0.6075949367088608,0.620253164556962,0.6708860759493671,0.7215189873417721,0.6550632911392404,0.8648211849820133
+claude-fable-5,federal_income_tax_before_refundable_credits,84,84,1.0,800.9046427408854,0.49174772344787193,,0.5714285714285714,0.5833333333333334,0.6309523809523809,0.6785714285714286,0.6160714285714286,0.8518397952830997
claude-fable-5,federal_refundable_credits,98,98,1.0,73.61601506447306,0.2161314886582061,,0.8775510204081632,0.9081632653061225,0.9285714285714286,0.9285714285714286,0.9107142857142858,0.945128098211834
claude-fable-5,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-claude-fable-5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-fable-5,payroll_tax,95,95,1.0,233.38396589740955,0.024829245574934294,,0.8631578947368421,0.8631578947368421,0.9263157894736842,0.9473684210526315,0.9,0.9845797316955671
+claude-fable-5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-fable-5,payroll_tax,96,96,1.0,231.97173617045084,0.02513145850714423,,0.8541666666666666,0.8541666666666666,0.9270833333333334,0.9479166666666666,0.8958333333333333,0.984292838433035
claude-fable-5,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514
claude-fable-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-fable-5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -14,14 +14,14 @@ claude-fable-5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0
claude-fable-5,self_employment_tax,100,100,1.0,34.48554270019532,0.0007099896483805804,,0.97,0.99,0.99,0.99,0.9850000000000001,0.9899432008281295
claude-fable-5,snap,93,93,1.0,262.7204301075269,0.5753977027817445,,0.8387096774193549,0.8387096774193549,0.8494623655913979,0.8709677419354839,0.8494623655913979,0.8873099985358851
claude-fable-5,ssi,97,97,1.0,123.63505154639175,0.19494688610780198,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.979381443298969,0.9742268041237113,0.9856711982245815
-claude-fable-5,state_income_tax_before_refundable_credits,85,85,1.0,211.68330901538627,0.4091439470844147,,0.6352941176470588,0.6470588235294118,0.7058823529411765,0.7294117647058823,0.6794117647058824,0.8306718289470787
-claude-fable-5,state_refundable_credits,98,98,1.0,67.9308757246757,0.7076828863448931,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8469387755102041,0.8341836734693878,0.8825146752432662
+claude-fable-5,state_income_tax_before_refundable_credits,87,87,1.0,290.9550223004133,0.38633732644394725,,0.6206896551724138,0.632183908045977,0.6896551724137931,0.7126436781609196,0.6637931034482758,0.8332853196113489
+claude-fable-5,state_refundable_credits,98,98,1.0,73.04871094470121,0.7090434056020384,,0.8367346938775511,0.8367346938775511,0.8469387755102041,0.8571428571428571,0.8443877551020409,0.8894999207429951
claude-fable-5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-fable-5.1,federal_income_tax_before_refundable_credits,79,79,1.0,258.39511842365505,0.15647206674704403,,0.8227848101265823,0.8481012658227848,0.9113924050632911,0.9113924050632911,0.8734177215189873,0.9415204647939512
+claude-fable-5.1,federal_income_tax_before_refundable_credits,84,84,1.0,137.81935070219492,0.0956733747993761,,0.7976190476190477,0.8452380952380952,0.9166666666666666,0.9166666666666666,0.869047619047619,0.9589971250859817
claude-fable-5.1,federal_refundable_credits,98,98,1.0,1.9756582703882253,0.005376964556959538,,0.9897959183673469,0.9897959183673469,0.9897959183673469,1.0,0.9923469387755102,0.9993964631619738
claude-fable-5.1,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-fable-5.1,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-fable-5.1,payroll_tax,95,95,1.0,101.24076935135692,0.011367186413134295,,0.9263157894736842,0.9473684210526315,0.9473684210526315,0.9578947368421052,0.9447368421052631,0.9929403789644745
+claude-fable-5.1,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-fable-5.1,payroll_tax,96,96,1.0,102.54513542175295,0.012835570670720338,,0.9166666666666666,0.9375,0.9375,0.9583333333333334,0.9375,0.9919777683307999
claude-fable-5.1,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786
claude-fable-5.1,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-fable-5.1,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -32,14 +32,14 @@ claude-fable-5.1,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99
claude-fable-5.1,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
claude-fable-5.1,snap,93,93,1.0,58.35526881720429,0.3985246550949543,,0.8817204301075269,0.9247311827956989,0.9247311827956989,0.9247311827956989,0.9139784946236559,0.9335395643415655
claude-fable-5.1,ssi,97,97,1.0,102.61855670103093,0.0,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846
-claude-fable-5.1,state_income_tax_before_refundable_credits,85,85,1.0,71.22791482880537,0.1081449221186077,,0.7411764705882353,0.788235294117647,0.8705882352941177,0.9058823529411765,0.8264705882352941,0.939574085394828
-claude-fable-5.1,state_refundable_credits,98,98,1.0,22.977769108013234,0.35719178258518913,,0.9183673469387755,0.9183673469387755,0.9183673469387755,0.9285714285714286,0.9209183673469388,0.9457682117915495
+claude-fable-5.1,state_income_tax_before_refundable_credits,87,87,1.0,68.5117336420081,0.09707350020825126,,0.735632183908046,0.7701149425287356,0.8505747126436781,0.896551724137931,0.8132183908045977,0.9424220861326921
+claude-fable-5.1,state_refundable_credits,98,98,1.0,28.48335943007956,0.21107292554413973,,0.9183673469387755,0.9183673469387755,0.9183673469387755,0.9285714285714286,0.9209183673469388,0.9429771455688737
claude-fable-5.1,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-haiku-4.5,federal_income_tax_before_refundable_credits,79,79,1.0,1172.859004153481,0.6717639782534175,,0.5443037974683544,0.5569620253164557,0.5949367088607594,0.620253164556962,0.5791139240506329,0.7351304026736567
+claude-haiku-4.5,federal_income_tax_before_refundable_credits,84,84,1.0,1707.3308657691591,0.6844398257930775,,0.5119047619047619,0.5357142857142857,0.5714285714285714,0.5952380952380952,0.5535714285714285,0.7127987524716456
claude-haiku-4.5,federal_refundable_credits,98,98,1.0,678.9802872015506,0.6425826245005123,,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.8054243992907588
claude-haiku-4.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-haiku-4.5,local_income_tax,100,100,1.0,35.45,,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-haiku-4.5,payroll_tax,95,95,1.0,961.6034026521379,0.14975173591769084,,0.6842105263157895,0.7473684210526316,0.7789473684210526,0.8315789473684211,0.7605263157894736,0.9030378553498539
+claude-haiku-4.5,local_income_tax,98,98,1.0,36.173469387755105,,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469
+claude-haiku-4.5,payroll_tax,96,96,1.0,953.9256579589843,0.1488996553382703,,0.6770833333333334,0.7395833333333334,0.7708333333333334,0.8333333333333334,0.7552083333333334,0.9030205141362884
claude-haiku-4.5,person_chip_eligible,177,177,1.0,0.096045197740113,,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887
claude-haiku-4.5,person_early_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579
claude-haiku-4.5,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -50,14 +50,32 @@ claude-haiku-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.07,,0.93,0.93
claude-haiku-4.5,self_employment_tax,100,100,1.0,74.96613400878907,0.0007522057448829237,,0.91,0.94,0.94,0.94,0.9325,0.9299398235404093
claude-haiku-4.5,snap,93,93,1.0,465.3333333333333,0.9093570110130051,,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8621328909336659
claude-haiku-4.5,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-claude-haiku-4.5,state_income_tax_before_refundable_credits,85,85,1.0,413.7844353940627,0.6679978305437163,,0.6352941176470588,0.6470588235294118,0.6705882352941176,0.6941176470588235,0.6617647058823528,0.758788505972342
-claude-haiku-4.5,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449
+claude-haiku-4.5,state_income_tax_before_refundable_credits,87,87,1.0,840.5786007846089,0.6544484706019632,,0.6206896551724138,0.632183908045977,0.6666666666666666,0.6781609195402298,0.6494252873563218,0.7538755189365801
+claude-haiku-4.5,state_refundable_credits,98,98,1.0,123.19856564366087,1.0,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898
claude-haiku-4.5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-opus-4.7,federal_income_tax_before_refundable_credits,79,79,1.0,661.971055429193,0.6648104124199187,,0.6329113924050633,0.6455696202531646,0.6708860759493671,0.7088607594936709,0.6645569620253164,0.8721264670595292
+claude-haiku-5.5,federal_income_tax_before_refundable_credits,84,84,1.0,1962.0004127139136,0.5693248396631865,,0.5476190476190477,0.5714285714285714,0.6190476190476191,0.6428571428571429,0.5952380952380952,0.7321941163348248
+claude-haiku-5.5,federal_refundable_credits,98,98,1.0,614.9173736572266,0.9866641641137136,,0.8877551020408163,0.8877551020408163,0.8877551020408163,0.8877551020408163,0.8877551020408163,0.889251981579073
+claude-haiku-5.5,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
+claude-haiku-5.5,local_income_tax,98,98,1.0,81.63265306122449,,,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939
+claude-haiku-5.5,payroll_tax,96,96,1.0,1782.4577042897542,0.27140627869968537,,0.6770833333333334,0.7083333333333334,0.7291666666666666,0.7604166666666666,0.71875,0.8315011362816822
+claude-haiku-5.5,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272
+claude-haiku-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
+claude-haiku-5.5,person_head_start_eligible,38,38,1.0,0.10526315789473684,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632
+claude-haiku-5.5,person_medicaid_eligible,173,173,1.0,0.10404624277456648,,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336
+claude-haiku-5.5,person_medicare_eligible,172,172,1.0,0.029069767441860465,,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395
+claude-haiku-5.5,person_wic_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028
+claude-haiku-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
+claude-haiku-5.5,self_employment_tax,100,100,1.0,0.16299572753906091,0.0017282267636608624,,0.96,1.0,1.0,1.0,0.99,0.9998617418589072
+claude-haiku-5.5,snap,93,93,1.0,521.6774193548387,0.9616024574427237,,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8655824521854257
+claude-haiku-5.5,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
+claude-haiku-5.5,state_income_tax_before_refundable_credits,87,87,1.0,1058.104978972468,0.6383930888083759,,0.6436781609195402,0.6551724137931034,0.6781609195402298,0.7011494252873564,0.6695402298850575,0.7725266005395443
+claude-haiku-5.5,state_refundable_credits,98,98,1.0,123.19856564366087,1.0,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898
+claude-haiku-5.5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
+claude-opus-4.7,federal_income_tax_before_refundable_credits,84,84,1.0,957.8269943963913,0.6224020290095342,,0.5952380952380952,0.6190476190476191,0.6309523809523809,0.6785714285714286,0.6309523809523809,0.8583409599806793
claude-opus-4.7,federal_refundable_credits,98,98,1.0,162.337958822445,0.2924270186642666,,0.826530612244898,0.8367346938775511,0.8469387755102041,0.8673469387755102,0.8443877551020409,0.8957479877009497
claude-opus-4.7,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-claude-opus-4.7,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-opus-4.7,payroll_tax,95,95,1.0,98.31385734477797,0.010193264299454344,,0.8736842105263158,0.8947368421052632,0.968421052631579,0.968421052631579,0.9263157894736842,0.993669446382444
+claude-opus-4.7,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-opus-4.7,payroll_tax,96,96,1.0,99.64871208190918,0.01168121392560172,,0.8645833333333334,0.8854166666666666,0.9583333333333334,0.96875,0.9192708333333334,0.9926992412964989
claude-opus-4.7,person_chip_eligible,177,177,1.0,0.11864406779661017,,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898
claude-opus-4.7,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-opus-4.7,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579
@@ -68,14 +86,14 @@ claude-opus-4.7,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,
claude-opus-4.7,self_employment_tax,100,100,1.0,0.022542163085938684,0.00020048201810881564,,1.0,1.0,1.0,1.0,1.0,0.9999839614385513
claude-opus-4.7,snap,93,93,1.0,465.18279569892474,1.8454323762656035,,0.7741935483870968,0.7741935483870968,0.7741935483870968,0.7956989247311828,0.7795698924731183,0.8296632759580936
claude-opus-4.7,ssi,97,97,1.0,166.9278350515464,0.3306139867067448,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9828739384184175
-claude-opus-4.7,state_income_tax_before_refundable_credits,85,85,1.0,180.86872755858477,0.2601958740816771,,0.6588235294117647,0.6941176470588235,0.7411764705882353,0.8352941176470589,0.7323529411764707,0.8876978782544869
-claude-opus-4.7,state_refundable_credits,98,98,1.0,77.61455355371747,4.132937401880446,,0.7653061224489796,0.7653061224489796,0.7755102040816326,0.7959183673469388,0.7755102040816326,0.8147258452245381
+claude-opus-4.7,state_income_tax_before_refundable_credits,87,87,1.0,225.28512848755406,0.26612601133329,,0.6436781609195402,0.6781609195402298,0.7126436781609196,0.8160919540229885,0.7126436781609196,0.8821849844674484
+claude-opus-4.7,state_refundable_credits,98,98,1.0,83.12014387578381,0.940675218714137,,0.7653061224489796,0.7653061224489796,0.7755102040816326,0.7959183673469388,0.7755102040816326,0.8139881952774096
claude-opus-4.7,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-opus-4.8,federal_income_tax_before_refundable_credits,79,79,1.0,838.9459985660601,0.9145477904016353,,0.5949367088607594,0.6329113924050633,0.6708860759493671,0.7341772151898734,0.6582278481012658,0.8184708035764722
+claude-opus-4.8,federal_income_tax_before_refundable_credits,84,84,1.0,1264.3116600399926,0.874262275276591,,0.5595238095238095,0.6071428571428571,0.6547619047619048,0.7023809523809523,0.6309523809523809,0.8046070941358859
claude-opus-4.8,federal_refundable_credits,98,98,1.0,198.6052529471261,0.3940920199472762,,0.8367346938775511,0.8367346938775511,0.8571428571428571,0.8571428571428571,0.8469387755102041,0.8945406916385711
claude-opus-4.8,free_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97
-claude-opus-4.8,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-opus-4.8,payroll_tax,95,95,1.0,306.0336558645148,0.02971467298377289,,0.8526315789473684,0.8842105263157894,0.9052631578947369,0.9263157894736842,0.8921052631578947,0.98154562414692
+claude-opus-4.8,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-opus-4.8,payroll_tax,96,96,1.0,305.20476270039876,0.030877265798514957,,0.84375,0.875,0.8958333333333334,0.9270833333333334,0.8854166666666667,0.9807017088759281
claude-opus-4.8,person_chip_eligible,177,177,1.0,0.12429378531073447,,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656
claude-opus-4.8,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
claude-opus-4.8,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579
@@ -86,14 +104,14 @@ claude-opus-4.8,reduced_price_school_meals_eligible,100,100,1.0,0.05,,0.95,0.95,
claude-opus-4.8,self_employment_tax,100,100,1.0,0.022667846679689775,0.0001990587122224831,,1.0,1.0,1.0,1.0,1.0,0.9999840753030221
claude-opus-4.8,snap,93,93,1.0,533.0860215053764,1.5275875662679301,,0.7634408602150538,0.7634408602150538,0.7634408602150538,0.7741935483870968,0.7661290322580645,0.8211884763998233
claude-opus-4.8,ssi,97,97,1.0,131.2164948453608,0.6589103291713961,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.9864142200170846
-claude-opus-4.8,state_income_tax_before_refundable_credits,85,85,1.0,236.9921737327576,0.28975293526220036,,0.6588235294117647,0.6588235294117647,0.7058823529411765,0.7411764705882353,0.6911764705882353,0.8658489985576021
-claude-opus-4.8,state_refundable_credits,98,98,1.0,53.63291890767156,1.455666318564968,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8086734693877551,0.831563420488021
+claude-opus-4.8,state_income_tax_before_refundable_credits,87,87,1.0,290.6392040645117,0.2643533506057612,,0.6436781609195402,0.6436781609195402,0.6781609195402298,0.735632183908046,0.6752873563218391,0.8713223693243839
+claude-opus-4.8,state_refundable_credits,98,98,1.0,58.75075412769707,1.482470219657025,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.826530612244898,0.8188775510204082,0.841767502120674
claude-opus-4.8,tanf,100,100,1.0,10.75041015625,0.17725445767969758,,0.99,0.99,0.99,0.99,0.99,0.998227455423203
-claude-opus-5,federal_income_tax_before_refundable_credits,79,79,1.0,551.3463212025316,0.5576880530290851,,0.5822784810126582,0.5822784810126582,0.6455696202531646,0.7088607594936709,0.629746835443038,0.843662505470099
+claude-opus-5,federal_income_tax_before_refundable_credits,84,84,1.0,943.8926449730283,0.559793695822973,,0.5476190476190477,0.5476190476190477,0.6071428571428571,0.6785714285714286,0.5952380952380953,0.825080840927771
claude-opus-5,federal_refundable_credits,98,98,1.0,195.23770048180404,0.16099216947942677,,0.8469387755102041,0.8571428571428571,0.8775510204081632,0.8877551020408163,0.8673469387755102,0.9309090422012888
claude-opus-5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-opus-5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-opus-5,payroll_tax,95,95,1.0,130.7998611482319,0.012762023755829034,,0.8947368421052632,0.9473684210526315,0.9578947368421052,0.9578947368421052,0.9394736842105262,0.9920741115621695
+claude-opus-5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-opus-5,payroll_tax,96,96,1.0,131.79840334574382,0.014208624857557384,,0.8854166666666666,0.9375,0.9479166666666666,0.9583333333333334,0.9322916666666666,0.9911196094640267
claude-opus-5,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272
claude-opus-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-opus-5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -104,14 +122,14 @@ claude-opus-5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.
claude-opus-5,self_employment_tax,100,100,1.0,54.23308532714844,0.0003572365859157118,,0.98,0.98,0.98,0.98,0.98,0.9799714210731267
claude-opus-5,snap,93,93,1.0,246.79569892473117,1.2430147066149062,,0.7849462365591398,0.7849462365591398,0.7956989247311828,0.8279569892473119,0.7983870967741935,0.8638796646667334
claude-opus-5,ssi,97,97,1.0,129.7319587628866,0.2856273584768493,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.983801497763364
-claude-opus-5,state_income_tax_before_refundable_credits,85,85,1.0,130.04105936005539,0.26529199878744375,,0.6823529411764706,0.6823529411764706,0.7647058823529411,0.8352941176470589,0.7411764705882353,0.9094886121784015
-claude-opus-5,state_refundable_credits,98,98,1.0,62.69297532159455,1.4088503474920302,,0.8163265306122449,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8290816326530612,0.841661214370422
+claude-opus-5,state_income_tax_before_refundable_credits,87,87,1.0,211.09885581334433,0.23055738869807335,,0.6781609195402298,0.6896551724137931,0.7586206896551724,0.8390804597701149,0.7413793103448275,0.917847367245514
+claude-opus-5,state_refundable_credits,98,98,1.0,68.19856564366087,0.8539529995116854,,0.8163265306122449,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8290816326530612,0.841661214370422
claude-opus-5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-opus-5.5,federal_income_tax_before_refundable_credits,79,79,1.0,268.9055683840981,0.1670578338164425,,0.8607594936708861,0.8860759493670886,0.9113924050632911,0.9113924050632911,0.8924050632911392,0.9373665561970987
+claude-opus-5.5,federal_income_tax_before_refundable_credits,84,84,1.0,156.29600527808773,0.09904482263065874,,0.8452380952380952,0.8928571428571429,0.9166666666666666,0.9285714285714286,0.8958333333333333,0.9575522188725747
claude-opus-5.5,federal_refundable_credits,98,98,1.0,0.003099128567447277,2.48862942154661e-05,,1.0,1.0,1.0,1.0,1.0,0.999997206640445
claude-opus-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-opus-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-opus-5.5,payroll_tax,95,95,1.0,10.23436217619245,0.00135439282676131,,0.968421052631579,0.9894736842105263,0.9894736842105263,1.0,0.9868421052631579,0.9991588507707484
+claude-opus-5.5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-opus-5.5,payroll_tax,96,96,1.0,12.486711654663104,0.0029896569774535706,,0.9583333333333334,0.9791666666666666,0.9791666666666666,1.0,0.9791666666666666,0.9981314643890915
claude-opus-5.5,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113
claude-opus-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-opus-5.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -122,14 +140,14 @@ claude-opus-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,
claude-opus-5.5,self_employment_tax,100,100,1.0,0.0015256591796855901,1.8684773766991133e-05,,1.0,1.0,1.0,1.0,1.0,0.9999985052180986
claude-opus-5.5,snap,93,93,1.0,85.5294623655914,0.5025349792380832,,0.8924731182795699,0.9139784946236559,0.9139784946236559,0.9139784946236559,0.9086021505376344,0.9318439754231114
claude-opus-5.5,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-opus-5.5,state_income_tax_before_refundable_credits,85,85,1.0,67.08857986786788,0.12525860302735892,,0.8117647058823529,0.8470588235294118,0.8823529411764706,0.9176470588235294,0.8647058823529411,0.9455000060259598
-claude-opus-5.5,state_refundable_credits,98,98,1.0,15.400220080589763,0.28047062896970437,,0.8979591836734694,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9056122448979591,0.9484849865157685
+claude-opus-5.5,state_income_tax_before_refundable_credits,87,87,1.0,57.310095430352234,0.10169072953671085,,0.8160919540229885,0.8390804597701149,0.8850574712643678,0.9310344827586207,0.867816091954023,0.9522711193604823
+claude-opus-5.5,state_refundable_credits,98,98,1.0,20.518055300615273,0.2540143128786343,,0.9081632653061225,0.9081632653061225,0.9183673469387755,0.9285714285714286,0.9158163265306123,0.9559362926639102
claude-opus-5.5,tanf,100,100,1.0,0.55041015625,0.009075249439740776,,0.99,1.0,1.0,1.0,0.9975,0.9999092475056025
-claude-sonnet-4.6,federal_income_tax_before_refundable_credits,79,79,1.0,364.404306146163,0.4699172731885661,,0.569620253164557,0.5949367088607594,0.7721518987341772,0.7974683544303798,0.6835443037974684,0.8950569435505298
+claude-sonnet-4.6,federal_income_tax_before_refundable_credits,84,84,1.0,554.0466773623511,0.44585889977718307,,0.5357142857142857,0.5595238095238095,0.75,0.7619047619047619,0.6517857142857143,0.8836430192542712
claude-sonnet-4.6,federal_refundable_credits,98,98,1.0,122.44227164132255,0.1746156152687253,,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9387755102040817,0.9158163265306123,0.9497880431841226
claude-sonnet-4.6,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-sonnet-4.6,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-sonnet-4.6,payroll_tax,95,95,1.0,46.291018528988445,0.004754896079615071,,0.8736842105263158,0.9052631578947369,1.0,1.0,0.9447368421052632,0.9970469592768706
+claude-sonnet-4.6,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-sonnet-4.6,payroll_tax,96,96,1.0,48.16777783711748,0.006333485176093102,,0.8645833333333334,0.8958333333333334,0.9895833333333334,1.0,0.9375,0.9960415717649417
claude-sonnet-4.6,person_chip_eligible,177,177,1.0,0.10734463276836158,,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384
claude-sonnet-4.6,person_early_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
claude-sonnet-4.6,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -140,14 +158,14 @@ claude-sonnet-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.9
claude-sonnet-4.6,self_employment_tax,100,100,1.0,34.44671838378906,0.00010697989103262154,,0.98,0.99,0.99,0.99,0.9875,0.9899914416087173
claude-sonnet-4.6,snap,93,93,1.0,257.6774193548387,0.5978746457112838,,0.8172043010752689,0.8172043010752689,0.8387096774193549,0.8602150537634409,0.8333333333333334,0.8744471957173431
claude-sonnet-4.6,ssi,97,97,1.0,10.68041237113402,0.05230402529537605,,0.979381443298969,0.979381443298969,0.9896907216494846,1.0,0.9871134020618557,0.9989215664887551
-claude-sonnet-4.6,state_income_tax_before_refundable_credits,85,85,1.0,184.57912163027595,0.3179950467965411,,0.6352941176470588,0.6588235294117647,0.7176470588235294,0.7647058823529411,0.6941176470588235,0.8749879326610083
-claude-sonnet-4.6,state_refundable_credits,98,98,1.0,62.84603654608435,0.9347312353332404,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8385187526938945
+claude-sonnet-4.6,state_income_tax_before_refundable_credits,87,87,1.0,209.41527766589462,0.2997262456221281,,0.6206896551724138,0.6436781609195402,0.7241379310344828,0.7816091954022989,0.6925287356321839,0.8770611151609128
+claude-sonnet-4.6,state_refundable_credits,98,98,1.0,65.65774931713027,0.9411136062034631,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8469496805565421
claude-sonnet-4.6,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-sonnet-5,federal_income_tax_before_refundable_credits,79,79,1.0,1114.880089868473,1.1430488761750437,,0.5189873417721519,0.5316455696202531,0.5443037974683544,0.5949367088607594,0.5474683544303797,0.7585010148292651
+claude-sonnet-5,federal_income_tax_before_refundable_credits,84,84,1.0,1706.5119687034971,1.0685440452061992,,0.4880952380952381,0.5,0.5238095238095238,0.5714285714285714,0.5208333333333333,0.739168673937573
claude-sonnet-5,federal_refundable_credits,98,98,1.0,257.36857106734294,0.8007227080206903,,0.7959183673469388,0.7959183673469388,0.8163265306122449,0.8163265306122449,0.8061224489795918,0.8579020748311923
claude-sonnet-5,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-claude-sonnet-5,local_income_tax,100,100,1.0,38.309,,,0.99,0.99,0.99,0.99,0.99,0.99
-claude-sonnet-5,payroll_tax,95,95,1.0,196.62585765316612,0.020444894782595972,,0.8,0.9052631578947369,0.9368421052631579,0.9473684210526315,0.8973684210526315,0.9873026442929143
+claude-sonnet-5,local_income_tax,98,98,1.0,39.090816326530614,,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469
+claude-sonnet-5,payroll_tax,96,96,1.0,196.93662905375163,0.02176198390069099,,0.7916666666666666,0.8958333333333334,0.9270833333333334,0.9479166666666666,0.890625,0.9863987600620682
claude-sonnet-5,person_chip_eligible,177,177,1.0,0.0847457627118644,,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356
claude-sonnet-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-sonnet-5,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
@@ -158,14 +176,14 @@ claude-sonnet-5,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0
claude-sonnet-5,self_employment_tax,100,100,1.0,82.20191687011719,0.06053951590470468,,0.96,0.96,0.96,0.96,0.96,0.9651568387276237
claude-sonnet-5,snap,93,93,1.0,650.2150537634409,2.0402639067599613,,0.7419354838709677,0.7419354838709677,0.7419354838709677,0.7526881720430108,0.7446236559139785,0.7837982352557759
claude-sonnet-5,ssi,97,97,1.0,141.5257731958763,0.6684457728437233,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.9862176129310573
-claude-sonnet-5,state_income_tax_before_refundable_credits,85,85,1.0,299.6872629446142,0.366121041319128,,0.6235294117647059,0.6352941176470588,0.6588235294117647,0.6941176470588235,0.6529411764705881,0.8280292917852385
-claude-sonnet-5,state_refundable_credits,98,98,1.0,82.61251074440625,4.366205870595451,,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8137755102040817,0.8220544240805049
+claude-sonnet-5,state_income_tax_before_refundable_credits,87,87,1.0,462.84148950138314,0.35875197474503195,,0.6091954022988506,0.6091954022988506,0.6436781609195402,0.6896551724137931,0.6379310344827587,0.8261918250908505
+claude-sonnet-5,state_refundable_credits,98,98,1.0,88.11810106647258,0.966694760816236,,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8137755102040817,0.8220544240805049
claude-sonnet-5,tanf,100,100,1.0,21.40958984375,0.35300469300628384,,0.99,0.99,0.99,0.99,0.99,0.9964699530699371
-claude-sonnet-5.5,federal_income_tax_before_refundable_credits,79,79,1.0,105.9342520025712,0.12697013459710574,,0.8354430379746836,0.8481012658227848,0.9113924050632911,0.9367088607594937,0.8829113924050633,0.9501762762973383
+claude-sonnet-5.5,federal_income_tax_before_refundable_credits,84,84,1.0,166.2165613374256,0.13206373477617295,,0.7976190476190477,0.8333333333333334,0.8928571428571429,0.9285714285714286,0.8630952380952381,0.9434012565244972
claude-sonnet-5.5,federal_refundable_credits,98,98,1.0,6.057284994319991,0.0020666673613282675,,0.9795918367346939,0.9795918367346939,0.9897959183673469,0.9897959183673469,0.9846938775510204,0.9895639455002591
claude-sonnet-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-claude-sonnet-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-sonnet-5.5,payroll_tax,95,95,1.0,12.62024015727796,0.0038158325055492027,,0.9578947368421052,0.9789473684210527,0.9789473684210527,0.9894736842105263,0.9763157894736842,0.9976301671807642
+claude-sonnet-5.5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+claude-sonnet-5.5,payroll_tax,96,96,1.0,14.847736740112305,0.005410072661594997,,0.9479166666666666,0.96875,0.96875,0.9895833333333334,0.96875,0.996618704586503
claude-sonnet-5.5,person_chip_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542
claude-sonnet-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
claude-sonnet-5.5,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -176,14 +194,14 @@ claude-sonnet-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.9
claude-sonnet-5.5,self_employment_tax,100,100,1.0,0.002925219726561318,2.0997990722224957e-05,,1.0,1.0,1.0,1.0,1.0,0.9899983201607424
claude-sonnet-5.5,snap,93,93,1.0,132.10752688172042,0.5926619058930392,,0.8602150537634409,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.8924731182795699,0.9102035545906004
claude-sonnet-5.5,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-claude-sonnet-5.5,state_income_tax_before_refundable_credits,85,85,1.0,73.5429514355379,0.26617039244015056,,0.7294117647058823,0.7529411764705882,0.8470588235294118,0.8588235294117647,0.7970588235294118,0.9194248309350876
-claude-sonnet-5.5,state_refundable_credits,98,98,1.0,80.80052383636942,1.105303908488078,,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.848005399455642
+claude-sonnet-5.5,state_income_tax_before_refundable_credits,87,87,1.0,99.46872343019507,0.26715903518805945,,0.7241379310344828,0.735632183908046,0.8275862068965517,0.8390804597701149,0.7816091954022988,0.914805999073758
+claude-sonnet-5.5,state_refundable_credits,98,98,1.0,85.91835905639493,1.11613716694365,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8574047720550642
claude-sonnet-5.5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-deepseek-v4-flash-0731,federal_income_tax_before_refundable_credits,79,79,1.0,726.5595648734178,0.7993699540504275,,0.5822784810126582,0.5949367088607594,0.7468354430379747,0.7468354430379747,0.6677215189873418,0.8512241931567712
+deepseek-v4-flash-0731,federal_income_tax_before_refundable_credits,84,84,1.0,823.9456939406622,0.683633907506447,,0.5476190476190477,0.5595238095238095,0.7142857142857143,0.7261904761904762,0.636904761904762,0.8555852012900012
deepseek-v4-flash-0731,federal_refundable_credits,98,98,1.0,275.2400379040776,0.1366504809061787,,0.826530612244898,0.8571428571428571,0.8979591836734694,0.8979591836734694,0.8698979591836735,0.9030290276533881
deepseek-v4-flash-0731,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-deepseek-v4-flash-0731,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-deepseek-v4-flash-0731,payroll_tax,95,95,1.0,265.36413377364306,0.04176946472448779,,0.8315789473684211,0.8526315789473684,0.9052631578947369,0.9157894736842105,0.8763157894736842,0.974058964013213
+deepseek-v4-flash-0731,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+deepseek-v4-flash-0731,payroll_tax,96,96,1.0,264.96096479797365,0.042732608476738494,,0.8229166666666666,0.84375,0.8958333333333334,0.9166666666666666,0.8697916666666666,0.9732921197020383
deepseek-v4-flash-0731,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272
deepseek-v4-flash-0731,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
deepseek-v4-flash-0731,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -194,14 +212,14 @@ deepseek-v4-flash-0731,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.9
deepseek-v4-flash-0731,self_employment_tax,100,100,1.0,34.44071120605469,0.00019954144430183585,,0.99,0.99,0.99,0.99,0.99,0.9799840366844559
deepseek-v4-flash-0731,snap,93,93,1.0,137.18494623655914,0.3493761853590002,,0.8279569892473119,0.8387096774193549,0.8924731182795699,0.9032258064516129,0.8655913978494624,0.9189044041971289
deepseek-v4-flash-0731,ssi,97,97,1.0,0.9896907216494846,0.004759119120710396,,0.979381443298969,1.0,1.0,1.0,0.9948453608247423,0.9999018738325628
-deepseek-v4-flash-0731,state_income_tax_before_refundable_credits,85,85,1.0,263.4360469519671,0.4403720535623172,,0.5294117647058824,0.5647058823529412,0.5882352941176471,0.6235294117647059,0.5764705882352942,0.7288219008520689
-deepseek-v4-flash-0731,state_refundable_credits,98,98,1.0,82.26940176107445,0.8221637663529808,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8403452641152473
+deepseek-v4-flash-0731,state_income_tax_before_refundable_credits,87,87,1.0,292.48138814093056,0.4191089650134316,,0.5172413793103449,0.5402298850574713,0.5862068965517241,0.632183908045977,0.5689655172413793,0.7325088874749043
+deepseek-v4-flash-0731,state_refundable_credits,98,98,1.0,87.38723698109996,0.811702811432568,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8505493457479004
deepseek-v4-flash-0731,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-deepseek-v4-pro,federal_income_tax_before_refundable_credits,79,79,1.0,705.1279662776899,0.5123312846761181,,0.5063291139240507,0.5189873417721519,0.6455696202531646,0.7088607594936709,0.5949367088607596,0.7910198223823275
+deepseek-v4-pro,federal_income_tax_before_refundable_credits,84,84,1.0,974.2758125232514,0.4595843829964634,,0.47619047619047616,0.4880952380952381,0.6190476190476191,0.6785714285714286,0.5654761904761905,0.7901720520658915
deepseek-v4-pro,federal_refundable_credits,98,98,1.0,108.02676143724092,0.3154062599117203,,0.8673469387755102,0.8775510204081632,0.8979591836734694,0.9183673469387755,0.8903061224489796,0.933985011642562
deepseek-v4-pro,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-deepseek-v4-pro,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-deepseek-v4-pro,payroll_tax,95,95,1.0,211.95588520250817,0.09656494296588437,,0.7052631578947368,0.7473684210526316,0.8421052631578947,0.8842105263157894,0.7947368421052632,0.9495323275337025
+deepseek-v4-pro,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+deepseek-v4-pro,payroll_tax,96,96,1.0,211.57655214945476,0.09624059630479163,,0.6979166666666666,0.7395833333333334,0.8333333333333334,0.8854166666666666,0.7890625,0.949254864295848
deepseek-v4-pro,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272
deepseek-v4-pro,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
deepseek-v4-pro,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -212,14 +230,14 @@ deepseek-v4-pro,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,
deepseek-v4-pro,self_employment_tax,100,100,1.0,34.42659978027344,1.906980687814305e-05,,0.99,0.99,0.99,0.99,0.99,0.9899984744154497
deepseek-v4-pro,snap,93,93,1.0,142.6658064516129,0.5539193918409281,,0.8494623655913979,0.8494623655913979,0.8817204301075269,0.9032258064516129,0.8709677419354839,0.9118177194200853
deepseek-v4-pro,ssi,97,97,1.0,7.051546391752577,0.03606961075014709,,0.979381443298969,0.9896907216494846,0.9896907216494846,1.0,0.9896907216494846,0.999256296685564
-deepseek-v4-pro,state_income_tax_before_refundable_credits,85,85,1.0,254.3113384845958,0.393232743553543,,0.5764705882352941,0.5764705882352941,0.6470588235294118,0.6823529411764706,0.6205882352941177,0.7960053555124469
-deepseek-v4-pro,state_refundable_credits,98,98,1.0,77.93138592875732,0.784702024008234,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8609095024121691
+deepseek-v4-pro,state_income_tax_before_refundable_credits,87,87,1.0,303.129259739799,0.37155222582125474,,0.5632183908045977,0.5632183908045977,0.6436781609195402,0.6896551724137931,0.6149425287356322,0.7993802963350787
+deepseek-v4-pro,state_refundable_credits,98,98,1.0,83.04922114878283,0.7766763480826386,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8316326530612246,0.870308875011591
deepseek-v4-pro,tanf,100,100,1.0,5.62958984375,0.09282156496446835,,0.99,0.99,0.99,1.0,0.9924999999999999,0.9990717843503554
-deepseek-v4-pro-0813,federal_income_tax_before_refundable_credits,79,79,1.0,862.1072507911393,0.7219055784554342,,0.5189873417721519,0.5189873417721519,0.5569620253164557,0.569620253164557,0.5411392405063291,0.7808340131677376
+deepseek-v4-pro-0813,federal_income_tax_before_refundable_credits,84,84,1.0,1060.3618565150668,0.6564489261091725,,0.4880952380952381,0.4880952380952381,0.5476190476190477,0.5595238095238095,0.5208333333333333,0.7821410760654579
deepseek-v4-pro-0813,federal_refundable_credits,98,98,1.0,83.4375667525311,0.28892672511029677,,0.9081632653061225,0.9285714285714286,0.9285714285714286,0.9489795918367347,0.9285714285714286,0.9675694492223136
deepseek-v4-pro-0813,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-deepseek-v4-pro-0813,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-deepseek-v4-pro-0813,payroll_tax,95,95,1.0,163.3633109837582,0.048518570794899844,,0.8315789473684211,0.8736842105263158,0.9157894736842105,0.9473684210526315,0.8921052631578946,0.9802650654264872
+deepseek-v4-pro-0813,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+deepseek-v4-pro-0813,payroll_tax,96,96,1.0,162.41535891215005,0.04823965069203657,,0.8229166666666666,0.8645833333333334,0.9166666666666666,0.9479166666666666,0.8880208333333333,0.9801395609468039
deepseek-v4-pro-0813,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
deepseek-v4-pro-0813,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
deepseek-v4-pro-0813,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -230,14 +248,14 @@ deepseek-v4-pro-0813,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.
deepseek-v4-pro-0813,self_employment_tax,100,100,1.0,0.0045822998046855905,2.7091941169829102e-05,,1.0,1.0,1.0,1.0,1.0,0.9999978326447063
deepseek-v4-pro-0813,snap,93,93,1.0,90.04215053763441,0.412657687654936,,0.8279569892473119,0.8387096774193549,0.9032258064516129,0.9032258064516129,0.8682795698924731,0.9100586028009229
deepseek-v4-pro-0813,ssi,97,97,1.0,1.6082474226804124,0.0076697449040421615,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9998418609298135
-deepseek-v4-pro-0813,state_income_tax_before_refundable_credits,85,85,1.0,142.79858390049657,0.24529904825068122,,0.6352941176470588,0.6941176470588235,0.7294117647058823,0.788235294117647,0.7117647058823529,0.8659637761269039
-deepseek-v4-pro-0813,state_refundable_credits,98,98,1.0,40.30240190307462,0.6197609478856888,,0.826530612244898,0.826530612244898,0.826530612244898,0.8367346938775511,0.8290816326530612,0.8759622748781388
+deepseek-v4-pro-0813,state_income_tax_before_refundable_credits,87,87,1.0,145.27693575677108,0.21047830933107575,,0.6206896551724138,0.7241379310344828,0.7471264367816092,0.8045977011494253,0.7241379310344828,0.8758133997792327
+deepseek-v4-pro-0813,state_refundable_credits,98,98,1.0,45.42023712310013,0.597393944820141,,0.8367346938775511,0.8367346938775511,0.8367346938775511,0.8469387755102041,0.8392857142857144,0.8861663565107919
deepseek-v4-pro-0813,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-deepseek-v4.1-flash,federal_income_tax_before_refundable_credits,79,79,1.0,342.60803992780853,0.23790061059837625,,0.6582278481012658,0.6708860759493671,0.7468354430379747,0.7721518987341772,0.7120253164556962,0.876587431619024
+deepseek-v4.1-flash,federal_income_tax_before_refundable_credits,84,84,1.0,363.15623893229167,0.18965629954066074,,0.6190476190476191,0.6547619047619048,0.7380952380952381,0.7619047619047619,0.6934523809523809,0.8857643339362712
deepseek-v4.1-flash,federal_refundable_credits,98,98,1.0,97.08988818110251,0.1129619008852461,,0.8979591836734694,0.9387755102040817,0.9591836734693877,0.9591836734693877,0.9387755102040817,0.977116521329207
deepseek-v4.1-flash,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-deepseek-v4.1-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-deepseek-v4.1-flash,payroll_tax,95,95,1.0,39.36758565481087,0.005532061389829545,,0.9052631578947369,0.9263157894736842,0.9789473684210527,0.9894736842105263,0.95,0.9965642987157902
+deepseek-v4.1-flash,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+deepseek-v4.1-flash,payroll_tax,96,96,1.0,41.31646405537924,0.007097697731137335,,0.8958333333333334,0.9166666666666666,0.96875,0.9895833333333334,0.9427083333333334,0.9955639389180391
deepseek-v4.1-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786
deepseek-v4.1-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
deepseek-v4.1-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -248,14 +266,14 @@ deepseek-v4.1-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0
deepseek-v4.1-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
deepseek-v4.1-flash,snap,93,93,1.0,171.45505376344084,0.5203532952046377,,0.8494623655913979,0.8924731182795699,0.9032258064516129,0.9032258064516129,0.8870967741935484,0.9078885966190541
deepseek-v4.1-flash,ssi,97,97,1.0,11.268041237113403,0.057990420209935334,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9988043212327848
-deepseek-v4.1-flash,state_income_tax_before_refundable_credits,85,85,1.0,76.04480081513348,0.11401921816618812,,0.6588235294117647,0.6823529411764706,0.7764705882352941,0.8588235294117647,0.7441176470588236,0.8905110902727122
-deepseek-v4.1-flash,state_refundable_credits,98,98,1.0,46.36871282052021,0.5478642572780168,,0.8367346938775511,0.8367346938775511,0.8469387755102041,0.8571428571428571,0.8443877551020409,0.8891677894795479
+deepseek-v4.1-flash,state_income_tax_before_refundable_credits,87,87,1.0,129.25676648874395,0.13342979129945898,,0.6436781609195402,0.6896551724137931,0.7701149425287356,0.8390804597701149,0.735632183908046,0.8834905341346755
+deepseek-v4.1-flash,state_refundable_credits,98,98,1.0,42.24165008136204,0.4846633129697711,,0.8469387755102041,0.8469387755102041,0.8571428571428571,0.8775510204081632,0.8571428571428571,0.9057216701991213
deepseek-v4.1-flash,tanf,100,100,1.0,18.224789843750003,0.300493208457008,,0.99,0.99,0.99,0.99,0.99,0.99699506791543
-gemini-3-flash-preview,federal_income_tax_before_refundable_credits,79,79,1.0,674.4463938884494,0.6128694955084341,,0.5569620253164557,0.569620253164557,0.6075949367088608,0.620253164556962,0.5886075949367089,0.7876260052079612
+gemini-3-flash-preview,federal_income_tax_before_refundable_credits,84,84,1.0,819.1213575381324,0.5345761336505803,,0.5238095238095238,0.5357142857142857,0.5714285714285714,0.6071428571428571,0.5595238095238095,0.7973412852472558
gemini-3-flash-preview,federal_refundable_credits,98,98,1.0,115.1862306555923,0.2283897327994623,,0.8571428571428571,0.8673469387755102,0.8775510204081632,0.9183673469387755,0.8801020408163265,0.9335480912163869
gemini-3-flash-preview,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3-flash-preview,local_income_tax,100,100,1.0,82.772,,,0.98,0.98,0.98,0.98,0.98,0.98
-gemini-3-flash-preview,payroll_tax,95,95,1.0,225.53196677117606,0.07058455043399657,,0.6842105263157895,0.6947368421052632,0.7894736842105263,0.8421052631578947,0.7526315789473684,0.9561632792041497
+gemini-3-flash-preview,local_income_tax,98,98,1.0,84.46122448979592,,,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939,0.9795918367346939
+gemini-3-flash-preview,payroll_tax,96,96,1.0,224.10621620178225,0.07005719085857348,,0.6770833333333334,0.6875,0.7916666666666666,0.84375,0.75,0.9562142557133916
gemini-3-flash-preview,person_chip_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758
gemini-3-flash-preview,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3-flash-preview,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -266,14 +284,14 @@ gemini-3-flash-preview,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.9
gemini-3-flash-preview,self_employment_tax,100,100,1.0,34.431631323242186,8.50016838390725e-05,,0.99,0.99,0.99,0.99,0.99,0.9899931998652929
gemini-3-flash-preview,snap,93,93,1.0,111.59784946236557,0.35513631357379194,,0.8172043010752689,0.8172043010752689,0.8709677419354839,0.8817204301075269,0.8467741935483871,0.9073465368122656
gemini-3-flash-preview,ssi,97,97,1.0,19.79381443298969,0.09693297442642909,,0.979381443298969,0.979381443298969,0.979381443298969,0.9896907216494846,0.9819587628865979,0.9980013819705891
-gemini-3-flash-preview,state_income_tax_before_refundable_credits,85,85,1.0,139.95913749941658,0.2774511804725019,,0.6352941176470588,0.6470588235294118,0.6941176470588235,0.7529411764705882,0.6823529411764706,0.8707180283483578
-gemini-3-flash-preview,state_refundable_credits,98,98,1.0,29.953633193385848,0.40578631341772303,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8571428571428571,0.8367346938775511,0.9152637383518468
+gemini-3-flash-preview,state_income_tax_before_refundable_credits,87,87,1.0,188.91196039813687,0.22965379257854912,,0.6206896551724138,0.6436781609195402,0.7241379310344828,0.7701149425287356,0.689655172413793,0.8843430927974475
+gemini-3-flash-preview,state_refundable_credits,98,98,1.0,34.5408561685134,0.3930363352579931,,0.8367346938775511,0.8367346938775511,0.8367346938775511,0.8571428571428571,0.8418367346938777,0.9216161459246338
gemini-3-flash-preview,tanf,100,100,1.0,16.45958984375,0.27138831253689305,,0.99,0.99,0.99,0.99,0.99,0.9972861168746312
-gemini-3.1-flash-lite-preview,federal_income_tax_before_refundable_credits,79,79,1.0,1258.1661763251582,0.9870901703123034,,0.4810126582278481,0.4936708860759494,0.5569620253164557,0.569620253164557,0.5253164556962026,0.6958929344589238
+gemini-3.1-flash-lite-preview,federal_income_tax_before_refundable_credits,84,84,1.0,1632.5257185872392,0.8803424056754334,,0.4523809523809524,0.4642857142857143,0.5238095238095238,0.5357142857142857,0.49404761904761907,0.6911332723018713
gemini-3.1-flash-lite-preview,federal_refundable_credits,98,98,1.0,272.9479859021245,0.5649095930538068,,0.8469387755102041,0.8469387755102041,0.8469387755102041,0.8673469387755102,0.8520408163265306,0.8957754538408992
gemini-3.1-flash-lite-preview,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.1-flash-lite-preview,local_income_tax,100,100,1.0,31.5,,,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.1-flash-lite-preview,payroll_tax,95,95,1.0,316.19232840768916,0.03118410248705525,,0.8,0.8842105263157894,0.8947368421052632,0.9368421052631579,0.8789473684210527,0.9806330310869867
+gemini-3.1-flash-lite-preview,local_income_tax,98,98,1.0,32.142857142857146,,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469
+gemini-3.1-flash-lite-preview,payroll_tax,96,96,1.0,315.2596990712484,0.03232366894326316,,0.7916666666666666,0.875,0.8854166666666666,0.9375,0.8723958333333333,0.9797977069104604
gemini-3.1-flash-lite-preview,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.1-flash-lite-preview,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
gemini-3.1-flash-lite-preview,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -284,14 +302,14 @@ gemini-3.1-flash-lite-preview,reduced_price_school_meals_eligible,100,100,1.0,0.
gemini-3.1-flash-lite-preview,self_employment_tax,100,100,1.0,20.57040251464844,0.0024292504799131066,,0.96,0.98,0.99,0.99,0.98,0.979805659961607
gemini-3.1-flash-lite-preview,snap,93,93,1.0,248.08602150537635,1.2318771910502693,,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8709677419354839,0.8548387096774194,0.8927662707850879
gemini-3.1-flash-lite-preview,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-gemini-3.1-flash-lite-preview,state_income_tax_before_refundable_credits,85,85,1.0,378.78839150619507,0.39023746315446867,,0.5411764705882353,0.5647058823529412,0.5882352941176471,0.6,0.573529411764706,0.7534452063733342
-gemini-3.1-flash-lite-preview,state_refundable_credits,98,98,1.0,114.71338348485986,0.9725137431284357,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8213750267723281
+gemini-3.1-flash-lite-preview,state_income_tax_before_refundable_credits,87,87,1.0,510.65326953909863,0.35607548950491613,,0.5287356321839081,0.5517241379310345,0.5862068965517241,0.5977011494252874,0.5660919540229885,0.7623148137765585
+gemini-3.1-flash-lite-preview,state_refundable_credits,98,98,1.0,119.83121870488537,0.9836539094569186,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8293661585635959
gemini-3.1-flash-lite-preview,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.1-pro-preview,federal_income_tax_before_refundable_credits,79,79,1.0,663.815019284019,0.44429262658154717,,0.5443037974683544,0.5569620253164557,0.5569620253164557,0.569620253164557,0.5569620253164557,0.7884916750200144
+gemini-3.1-pro-preview,federal_income_tax_before_refundable_credits,84,84,1.0,745.5544788178943,0.3828785109995266,,0.5119047619047619,0.5238095238095238,0.5357142857142857,0.5476190476190477,0.5297619047619047,0.796982154074579
gemini-3.1-pro-preview,federal_refundable_credits,98,98,1.0,92.27329204948582,0.29286136212849573,,0.8673469387755102,0.8979591836734694,0.8979591836734694,0.9183673469387755,0.8954081632653061,0.9365155613937403
gemini-3.1-pro-preview,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.1-pro-preview,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.1-pro-preview,payroll_tax,95,95,1.0,221.81594153474504,0.09481823498553894,,0.6842105263157895,0.6947368421052632,0.7789473684210526,0.8210526315789474,0.7447368421052631,0.9411128856405601
+gemini-3.1-pro-preview,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gemini-3.1-pro-preview,payroll_tax,96,96,1.0,220.4288995615641,0.09388698066759014,,0.6770833333333334,0.6875,0.78125,0.8229166666666666,0.7421875,0.9413206370827562
gemini-3.1-pro-preview,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272
gemini-3.1-pro-preview,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.1-pro-preview,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
@@ -302,14 +320,14 @@ gemini-3.1-pro-preview,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,
gemini-3.1-pro-preview,self_employment_tax,100,100,1.0,0.011282299804687455,0.000145894883346029,,1.0,1.0,1.0,1.0,1.0,0.9999883284093323
gemini-3.1-pro-preview,snap,93,93,1.0,120.24946236559141,0.4976533739500507,,0.8387096774193549,0.8387096774193549,0.8924731182795699,0.8924731182795699,0.8655913978494624,0.908930173533864
gemini-3.1-pro-preview,ssi,97,97,1.0,110.84536082474227,0.06619001968689023,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.9896907216494846,0.9768041237113402,0.9883259789755281
-gemini-3.1-pro-preview,state_income_tax_before_refundable_credits,85,85,1.0,176.86964196575389,0.27863797547807384,,0.6588235294117647,0.6705882352941176,0.7294117647058823,0.7764705882352941,0.7088235294117646,0.8806759857911499
-gemini-3.1-pro-preview,state_refundable_credits,98,98,1.0,54.76942216620154,0.5418215330691546,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8800735959668899
+gemini-3.1-pro-preview,state_income_tax_before_refundable_credits,87,87,1.0,184.95014420235293,0.23237380520655626,,0.6436781609195402,0.6781609195402298,0.7241379310344828,0.7816091954022989,0.7068965517241379,0.8934985301115935
+gemini-3.1-pro-preview,state_refundable_credits,98,98,1.0,59.88725738622704,0.5233745285224686,,0.8367346938775511,0.8367346938775511,0.8367346938775511,0.8367346938775511,0.8367346938775511,0.888802377705286
gemini-3.1-pro-preview,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.5-flash,federal_income_tax_before_refundable_credits,79,79,1.0,895.530635012856,0.6208179584634841,,0.5316455696202531,0.5443037974683544,0.569620253164557,0.5822784810126582,0.5569620253164557,0.7672878326511589
+gemini-3.5-flash,federal_income_tax_before_refundable_credits,84,84,1.0,939.3459879557291,0.5187684788586684,,0.5,0.5119047619047619,0.5476190476190477,0.5714285714285714,0.5327380952380952,0.782412612081685
gemini-3.5-flash,federal_refundable_credits,98,98,1.0,108.31145531479194,0.2475640087124831,,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.9183673469387755,0.8775510204081632,0.9313958765730888
gemini-3.5-flash,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.5-flash,local_income_tax,100,100,1.0,28.1812,,,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.5-flash,payroll_tax,95,95,1.0,272.38419153474507,0.11517458700948496,,0.6421052631578947,0.6526315789473685,0.7157894736842105,0.7894736842105263,0.7,0.928470519646741
+gemini-3.5-flash,local_income_tax,98,98,1.0,28.756326530612245,,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469
+gemini-3.5-flash,payroll_tax,96,96,1.0,270.6162302907308,0.11400654948090913,,0.6354166666666666,0.6458333333333334,0.71875,0.7916666666666666,0.6979166666666666,0.9287459065744317
gemini-3.5-flash,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113
gemini-3.5-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.5-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -320,14 +338,14 @@ gemini-3.5-flash,reduced_price_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97
gemini-3.5-flash,self_employment_tax,100,100,1.0,0.008882299804683953,0.00014162740073649118,,1.0,1.0,1.0,1.0,1.0,0.9999886698079411
gemini-3.5-flash,snap,93,93,1.0,86.02150537634408,0.275273051861285,,0.8387096774193549,0.8494623655913979,0.9032258064516129,0.9139784946236559,0.8763440860215054,0.9400155949011109
gemini-3.5-flash,ssi,97,97,1.0,5.072164948453608,0.025894900504143357,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9994660845256877
-gemini-3.5-flash,state_income_tax_before_refundable_credits,85,85,1.0,216.2225603940627,0.4080018888248032,,0.611764705882353,0.6352941176470588,0.6705882352941176,0.7058823529411765,0.6558823529411765,0.8314446103001986
-gemini-3.5-flash,state_refundable_credits,98,98,1.0,41.819855690197066,0.6144782376127053,,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8137755102040817,0.876932568601748
+gemini-3.5-flash,state_income_tax_before_refundable_credits,87,87,1.0,218.2493973004133,0.35527776049883525,,0.5977011494252874,0.632183908045977,0.6781609195402298,0.7471264367816092,0.6637931034482759,0.8447268514479572
+gemini-3.5-flash,state_refundable_credits,98,98,1.0,40.3458541755287,0.5730999875451223,,0.8163265306122449,0.826530612244898,0.826530612244898,0.826530612244898,0.8239795918367346,0.8903806144054379
gemini-3.5-flash,tanf,100,100,1.0,7.66958984375,0.1264574066124597,,0.99,0.99,0.99,0.99,0.99,0.9987354259338754
-gemini-3.5-flash-lite,federal_income_tax_before_refundable_credits,79,79,1.0,1050.4442679489716,0.6445252374949046,,0.569620253164557,0.5822784810126582,0.620253164556962,0.6582278481012658,0.6075949367088607,0.7688366584094108
+gemini-3.5-flash-lite,federal_income_tax_before_refundable_credits,84,84,1.0,1598.2836361839659,0.6571879721502071,,0.5357142857142857,0.5476190476190477,0.5714285714285714,0.6190476190476191,0.5684523809523809,0.7469990423251135
gemini-3.5-flash-lite,federal_refundable_credits,98,98,1.0,302.53982263681837,0.6900239011584266,,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.9123442559924215
gemini-3.5-flash-lite,free_school_meals_eligible,100,100,1.0,0.04,,0.96,0.96,0.96,0.96,0.96,0.96,0.96
-gemini-3.5-flash-lite,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.5-flash-lite,payroll_tax,95,95,1.0,544.1144432308798,0.06631418999262825,,0.8210526315789474,0.8526315789473684,0.8842105263157894,0.9052631578947369,0.8657894736842106,0.958815397794052
+gemini-3.5-flash-lite,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gemini-3.5-flash-lite,payroll_tax,96,96,1.0,540.8055418650309,0.06686679085722272,,0.8125,0.84375,0.875,0.90625,0.859375,0.9582082557142358
gemini-3.5-flash-lite,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757
gemini-3.5-flash-lite,person_early_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
gemini-3.5-flash-lite,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -338,14 +356,14 @@ gemini-3.5-flash-lite,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1
gemini-3.5-flash-lite,self_employment_tax,100,100,1.0,48.990022631835934,0.33281306928834004,,0.95,0.97,0.97,0.97,0.9649999999999999,0.9633749544569329
gemini-3.5-flash-lite,snap,93,93,1.0,351.2043010752688,0.8279607700662017,,0.8602150537634409,0.8602150537634409,0.8817204301075269,0.8817204301075269,0.8709677419354839,0.884263548270316
gemini-3.5-flash-lite,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-gemini-3.5-flash-lite,state_income_tax_before_refundable_credits,85,85,1.0,411.31406361097453,0.5181781747675703,,0.6235294117647059,0.6235294117647059,0.6352941176470588,0.6705882352941176,0.638235294117647,0.7911149597162025
-gemini-3.5-flash-lite,state_refundable_credits,98,98,1.0,99.71338348485986,0.9733517660885691,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8212211041878138
+gemini-3.5-flash-lite,state_income_tax_before_refundable_credits,87,87,1.0,521.3827939419364,0.4923539710109554,,0.6091954022988506,0.6091954022988506,0.632183908045977,0.6666666666666666,0.6293103448275862,0.7932065004919212
+gemini-3.5-flash-lite,state_refundable_credits,98,98,1.0,104.83121870488537,0.9717842229173085,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8314251858204669
gemini-3.5-flash-lite,tanf,100,100,1.0,118.64958984375,1.0,,0.98,0.98,0.98,0.98,0.98,0.98
-gemini-3.6-flash,federal_income_tax_before_refundable_credits,79,79,1.0,523.4174938192247,0.3546634078463114,,0.5822784810126582,0.5949367088607594,0.6708860759493671,0.7215189873417721,0.6424050632911392,0.8726305394284314
+gemini-3.6-flash,federal_income_tax_before_refundable_credits,84,84,1.0,501.63606666201633,0.2726917812312435,,0.5476190476190477,0.5595238095238095,0.6547619047619048,0.7023809523809523,0.6160714285714286,0.8797859145568795
gemini-3.6-flash,federal_refundable_credits,98,98,1.0,61.924495961714754,0.15370798512544603,,0.8979591836734694,0.9183673469387755,0.9489795918367347,0.9591836734693877,0.9311224489795918,0.9725429812614295
gemini-3.6-flash,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.6-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.6-flash,payroll_tax,95,95,1.0,180.10952289782068,0.04884688856323825,,0.7263157894736842,0.7368421052631579,0.8315789473684211,0.8736842105263158,0.7921052631578948,0.9696635113133575
+gemini-3.6-flash,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gemini-3.6-flash,payroll_tax,96,96,1.0,179.15463111877435,0.04868021280582185,,0.71875,0.7291666666666666,0.8333333333333334,0.875,0.7890625,0.9695748669963615
gemini-3.6-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786
gemini-3.6-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.6-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -356,14 +374,14 @@ gemini-3.6-flash,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.
gemini-3.6-flash,self_employment_tax,100,100,1.0,0.0027822998046872272,2.518602137197596e-05,,1.0,1.0,1.0,1.0,1.0,0.9999979851182903
gemini-3.6-flash,snap,93,93,1.0,99.29032258064517,0.29846023082180606,,0.8494623655913979,0.8494623655913979,0.9032258064516129,0.9032258064516129,0.8763440860215054,0.9475270645087798
gemini-3.6-flash,ssi,97,97,1.0,1.731958762886598,0.008747871130837852,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9998196315230755
-gemini-3.6-flash,state_income_tax_before_refundable_credits,85,85,1.0,155.7966909231298,0.7359836180256886,,0.6352941176470588,0.6705882352941176,0.7529411764705882,0.7647058823529411,0.7058823529411764,0.8672795245534004
-gemini-3.6-flash,state_refundable_credits,98,98,1.0,43.578893688941484,0.6382126164966077,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8725731928883781
+gemini-3.6-flash,state_income_tax_before_refundable_credits,87,87,1.0,159.11901611832366,0.6784471926177192,,0.6206896551724138,0.6551724137931034,0.7586206896551724,0.7701149425287356,0.7011494252873562,0.8739128912486749
+gemini-3.6-flash,state_refundable_credits,98,98,1.0,48.696728908966996,0.629688010670277,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8316326530612246,0.8805643246796458
gemini-3.6-flash,tanf,100,100,1.0,7.664789843750004,0.12637826345564096,,0.99,0.99,0.99,0.99,0.99,0.9987362173654435
-gemini-3.7-flash,federal_income_tax_before_refundable_credits,79,79,1.0,741.4982052264636,0.63052384021866,,0.569620253164557,0.5822784810126582,0.6329113924050633,0.6582278481012658,0.610759493670886,0.8101198010284637
+gemini-3.7-flash,federal_income_tax_before_refundable_credits,84,84,1.0,788.0746198381696,0.5063360395353433,,0.5357142857142857,0.5476190476190477,0.6071428571428571,0.6547619047619048,0.5863095238095238,0.8219003850427068
gemini-3.7-flash,federal_refundable_credits,98,98,1.0,116.61798592703684,0.3015668492484629,,0.8571428571428571,0.8775510204081632,0.9081632653061225,0.9183673469387755,0.8903061224489796,0.9253343332476215
gemini-3.7-flash,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.7-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.7-flash,payroll_tax,95,95,1.0,192.01041136410365,0.08655841672327612,,0.7368421052631579,0.7368421052631579,0.8210526315789474,0.8526315789473684,0.7868421052631579,0.9462426675087021
+gemini-3.7-flash,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gemini-3.7-flash,payroll_tax,96,96,1.0,190.9180103302002,0.08575369863080881,,0.7291666666666666,0.7291666666666666,0.8229166666666666,0.8541666666666666,0.7838541666666666,0.9464039383557447
gemini-3.7-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786
gemini-3.7-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.7-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -374,14 +392,14 @@ gemini-3.7-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99
gemini-3.7-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
gemini-3.7-flash,snap,93,93,1.0,153.13978494623655,0.4787112113664552,,0.8387096774193549,0.8387096774193549,0.8817204301075269,0.8817204301075269,0.8602150537634409,0.9155966841502522
gemini-3.7-flash,ssi,97,97,1.0,11.257731958762887,0.0577729290234779,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9988088055871448
-gemini-3.7-flash,state_income_tax_before_refundable_credits,85,85,1.0,167.11546733340094,0.32139198477059217,,0.6941176470588235,0.7294117647058823,0.7764705882352941,0.8,0.75,0.880254564746198
-gemini-3.7-flash,state_refundable_credits,98,98,1.0,33.076565023344386,0.5133737012636046,,0.8367346938775511,0.826530612244898,0.8469387755102041,0.8571428571428571,0.8418367346938777,0.9062100282421046
+gemini-3.7-flash,state_income_tax_before_refundable_credits,87,87,1.0,213.55371191068625,0.27693008877546726,,0.6781609195402298,0.7126436781609196,0.7701149425287356,0.8160919540229885,0.7442528735632183,0.89146175643373
+gemini-3.7-flash,state_refundable_credits,98,98,1.0,38.1944002433699,0.49750562983533253,,0.8469387755102041,0.8367346938775511,0.8571428571428571,0.8673469387755102,0.8520408163265306,0.9142011600333722
gemini-3.7-flash,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gemini-3.8-flash,federal_income_tax_before_refundable_credits,79,79,1.0,477.05485215585446,0.3861861161847889,,0.5949367088607594,0.5949367088607594,0.7215189873417721,0.7848101265822784,0.6740506329113924,0.8923971417076991
+gemini-3.8-flash,federal_income_tax_before_refundable_credits,84,84,1.0,368.8653456333705,0.27731129721445696,,0.5595238095238095,0.5714285714285714,0.7142857142857143,0.7976190476190477,0.6607142857142858,0.9160344679950341
gemini-3.8-flash,federal_refundable_credits,98,98,1.0,80.45643473722498,0.1819363933843249,,0.9081632653061225,0.9285714285714286,0.9387755102040817,0.9591836734693877,0.9336734693877551,0.9795785680895146
gemini-3.8-flash,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gemini-3.8-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gemini-3.8-flash,payroll_tax,95,95,1.0,130.28012708162007,0.017115981238638493,,0.8526315789473684,0.8842105263157894,0.9578947368421052,0.9578947368421052,0.9131578947368421,0.9893700748096879
+gemini-3.8-flash,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gemini-3.8-flash,payroll_tax,96,96,1.0,131.28199984232583,0.018488552249132803,,0.84375,0.875,0.9479166666666666,0.9583333333333334,0.90625,0.988444654844292
gemini-3.8-flash,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028
gemini-3.8-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gemini-3.8-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -392,14 +410,14 @@ gemini-3.8-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99
gemini-3.8-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
gemini-3.8-flash,snap,93,93,1.0,52.98924731182796,0.41794495554072547,,0.8494623655913979,0.8494623655913979,0.9247311827956989,0.9247311827956989,0.8870967741935484,0.9308248986878556
gemini-3.8-flash,ssi,97,97,1.0,8.412371134020619,0.040561924526802184,,0.979381443298969,0.979381443298969,1.0,1.0,0.9896907216494846,0.9991636716592412
-gemini-3.8-flash,state_income_tax_before_refundable_credits,85,85,1.0,167.57604897689822,0.19424530663659081,,0.7058823529411765,0.7411764705882353,0.8,0.8235294117647058,0.7676470588235293,0.8866692483239867
-gemini-3.8-flash,state_refundable_credits,98,98,1.0,32.65154933909981,0.4995806944963879,,0.8571428571428571,0.8469387755102041,0.8673469387755102,0.8673469387755102,0.8596938775510203,0.9082402806027043
+gemini-3.8-flash,state_income_tax_before_refundable_credits,87,87,1.0,186.1388231069192,0.1344479396251096,,0.6896551724137931,0.7241379310344828,0.8160919540229885,0.8505747126436781,0.7701149425287357,0.9061162513979496
+gemini-3.8-flash,state_refundable_credits,98,98,1.0,37.769384559125314,0.47478317565833095,,0.8673469387755102,0.8571428571428571,0.8775510204081632,0.8775510204081632,0.8698979591836734,0.9176396532021263
gemini-3.8-flash,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-glm-5.2,federal_income_tax_before_refundable_credits,79,73,0.9240506329113924,494.96764126712327,0.3813424597334745,,0.5063291139240507,0.5063291139240507,0.6708860759493671,0.7215189873417722,0.6012658227848102,0.8103477999545824
+glm-5.2,federal_income_tax_before_refundable_credits,84,76,0.9047619047619048,852.3193473735608,0.3760450732776106,,0.47619047619047616,0.47619047619047616,0.6547619047619048,0.7142857142857143,0.5803571428571429,0.7887984082970579
glm-5.2,federal_refundable_credits,98,88,0.8979591836734694,155.9154522566362,0.4696846776647093,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8337980332280309
glm-5.2,free_school_meals_eligible,100,95,0.95,0.021052631578947368,,0.93,0.93,0.93,0.93,0.93,0.93,0.93
-glm-5.2,local_income_tax,100,95,0.95,37.4,,,0.94,0.94,0.94,0.94,0.94,0.94
-glm-5.2,payroll_tax,95,87,0.9157894736842105,163.13953463148795,0.049616676528711524,,0.7578947368421052,0.8105263157894737,0.863157894736842,0.8736842105263157,0.8263157894736842,0.8985062424883542
+glm-5.2,local_income_tax,98,93,0.9489795918367347,38.204301075268816,,,0.9387755102040817,0.9387755102040817,0.9387755102040817,0.9387755102040817,0.9387755102040817,0.9387755102040817
+glm-5.2,payroll_tax,96,87,0.90625,163.13953463148795,0.049616676528711524,,0.75,0.8020833333333334,0.8541666666666666,0.8645833333333333,0.8177083333333333,0.8891468024624339
glm-5.2,person_chip_eligible,177,165,0.9322033898305084,0.0,,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084
glm-5.2,person_early_head_start_eligible,38,34,0.8947368421052632,0.0,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632
glm-5.2,person_head_start_eligible,38,34,0.8947368421052632,0.0,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632
@@ -410,14 +428,14 @@ glm-5.2,reduced_price_school_meals_eligible,100,95,0.95,0.010526315789473684,,0.
glm-5.2,self_employment_tax,100,92,0.92,40.462241555918816,3.352336279141471e-05,,0.91,0.91,0.91,0.91,0.91,0.8999976533646046
glm-5.2,snap,93,89,0.956989247311828,160.2334831460674,1.1696998867207566,,0.8172043010752689,0.8279569892473119,0.8602150537634409,0.881720430107527,0.8467741935483871,0.8828924267797433
glm-5.2,ssi,97,92,0.9484536082474226,42.84804347826087,0.1165670367207515,,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9369426078688582
-glm-5.2,state_income_tax_before_refundable_credits,85,78,0.9176470588235294,288.8558082362933,1.254862054801632,,0.5294117647058822,0.5411764705882353,0.6117647058823529,0.6470588235294118,0.5823529411764705,0.7218708317198196
-glm-5.2,state_refundable_credits,98,90,0.9183673469387755,45.22765429963006,0.9019214373939038,,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7603997799907289
+glm-5.2,state_income_tax_before_refundable_credits,87,79,0.9080459770114943,513.4173275001141,1.3756690897919994,,0.5172413793103449,0.5402298850574713,0.5862068965517241,0.632183908045977,0.5689655172413793,0.7120207247059817
+glm-5.2,state_refundable_credits,98,90,0.9183673469387755,45.01654318851895,0.8949158257791827,,0.7551020408163265,0.7551020408163265,0.7551020408163265,0.7551020408163265,0.7551020408163265,0.7706038616233819
glm-5.2,tanf,100,95,0.95,63.841673519736844,1.0,,0.94,0.94,0.94,0.94,0.94,0.94
-glm-5.3,federal_income_tax_before_refundable_credits,79,77,0.9746835443037974,480.8747809709822,0.45871447509940255,,0.5822784810126582,0.620253164556962,0.7088607594936709,0.7468354430379747,0.6645569620253164,0.8261926138728213
+glm-5.3,federal_income_tax_before_refundable_credits,84,81,0.9642857142857143,643.1087116608796,0.4091371037145281,,0.5476190476190476,0.5952380952380952,0.6785714285714286,0.7261904761904762,0.6369047619047619,0.8218275170394038
glm-5.3,federal_refundable_credits,98,96,0.9795918367346939,253.52654483795166,0.34476172707133146,,0.826530612244898,0.836734693877551,0.8673469387755102,0.8775510204081632,0.8520408163265306,0.895311606335925
glm-5.3,free_school_meals_eligible,100,97,0.97,0.020618556701030927,,0.95,0.95,0.95,0.95,0.95,0.95,0.95
-glm-5.3,local_income_tax,100,97,0.97,0.0,,,0.97,0.97,0.97,0.97,0.97,0.97
-glm-5.3,payroll_tax,95,92,0.968421052631579,200.42261631177823,0.02771657924639223,,0.8526315789473684,0.8842105263157894,0.9052631578947369,0.9157894736842105,0.8894736842105264,0.9415565427600215
+glm-5.3,local_income_tax,98,95,0.9693877551020408,0.0,,,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408
+glm-5.3,payroll_tax,96,93,0.96875,200.70258723433307,0.028975415432741437,,0.84375,0.875,0.8958333333333333,0.9166666666666667,0.8828125,0.9411291804201433
glm-5.3,person_chip_eligible,177,168,0.9491525423728814,0.017857142857142856,,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084
glm-5.3,person_early_head_start_eligible,38,35,0.9210526315789473,0.0,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473
glm-5.3,person_head_start_eligible,38,35,0.9210526315789473,0.0,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473
@@ -428,14 +446,14 @@ glm-5.3,reduced_price_school_meals_eligible,100,97,0.97,0.020618556701030927,,0.
glm-5.3,self_employment_tax,100,97,0.97,0.00884773175740759,0.00010957748686821732,,0.97,0.97,0.97,0.97,0.97,0.9699912338010506
glm-5.3,snap,93,90,0.967741935483871,116.08888888888889,0.49502346382348883,,0.8064516129032259,0.8387096774193549,0.8602150537634409,0.8709677419354839,0.8440860215053764,0.8789404585705626
glm-5.3,ssi,97,95,0.979381443298969,102.6842105263158,0.0005675368898978433,,0.9587628865979382,0.9690721649484535,0.9690721649484535,0.9690721649484535,0.9664948453608246,0.9690604631569093
-glm-5.3,state_income_tax_before_refundable_credits,85,84,0.9882352941176471,217.95254043374743,0.7836490142289976,,0.5764705882352942,0.6352941176470589,0.6823529411764706,0.6823529411764706,0.6441176470588236,0.805799974322545
-glm-5.3,state_refundable_credits,98,95,0.9693877551020408,67.42412189463565,1.176670574544347,,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.8298857836589686
+glm-5.3,state_income_tax_before_refundable_credits,87,85,0.9770114942528736,272.38049664463716,0.7886244619949137,,0.5632183908045977,0.6206896551724138,0.6666666666666667,0.6781609195402298,0.632183908045977,0.7954812414558727
+glm-5.3,state_refundable_credits,98,95,0.9693877551020408,62.1446708255567,0.7366911840962915,,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.8313903047996228
glm-5.3,tanf,100,97,0.97,235.72122664304123,1.0,,0.94,0.94,0.94,0.94,0.94,0.94
-gpt-5.4-mini,federal_income_tax_before_refundable_credits,79,79,1.0,1332.7757120253164,0.7629104866864134,,0.5822784810126582,0.5822784810126582,0.5822784810126582,0.5949367088607594,0.5854430379746836,0.6753136064901415
+gpt-5.4-mini,federal_income_tax_before_refundable_credits,84,84,1.0,2091.4508201962426,0.7643966580293052,,0.5476190476190477,0.5476190476190477,0.5476190476190477,0.5595238095238095,0.5505952380952381,0.648591908463631
gpt-5.4-mini,federal_refundable_credits,98,98,1.0,514.8456610854792,0.7891013249460958,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8502029125060504
gpt-5.4-mini,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gpt-5.4-mini,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.4-mini,payroll_tax,95,95,1.0,1047.0798554995888,0.2561243232296806,,0.5684210526315789,0.6842105263157895,0.6947368421052632,0.7473684210526316,0.6736842105263158,0.8304069992573563
+gpt-5.4-mini,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.4-mini,payroll_tax,96,96,1.0,1055.1346495869955,0.26518168618272664,,0.5625,0.6770833333333334,0.6875,0.7395833333333334,0.6666666666666667,0.8238447794691292
gpt-5.4-mini,person_chip_eligible,177,177,1.0,0.07909604519774012,,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598
gpt-5.4-mini,person_early_head_start_eligible,38,38,1.0,0.07894736842105263,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473
gpt-5.4-mini,person_head_start_eligible,38,38,1.0,0.15789473684210525,,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947
@@ -446,14 +464,14 @@ gpt-5.4-mini,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.
gpt-5.4-mini,self_employment_tax,100,100,1.0,73.5364168701172,0.08842207254988142,,0.9,0.95,0.95,0.95,0.9375,0.9629262341960094
gpt-5.4-mini,snap,93,93,1.0,546.5591397849462,1.3854698063203785,,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.8215651525215243
gpt-5.4-mini,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-gpt-5.4-mini,state_income_tax_before_refundable_credits,85,85,1.0,462.8933549712686,0.6761316167730925,,0.6470588235294118,0.6470588235294118,0.6705882352941176,0.6823529411764706,0.661764705882353,0.7575550954538861
-gpt-5.4-mini,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449
+gpt-5.4-mini,state_income_tax_before_refundable_credits,87,87,1.0,847.0572957828127,0.683278044833453,,0.632183908045977,0.632183908045977,0.6551724137931034,0.6666666666666666,0.646551724137931,0.74503885758808
+gpt-5.4-mini,state_refundable_credits,98,98,1.0,123.19856564366087,1.0,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898
gpt-5.4-mini,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gpt-5.4-nano,federal_income_tax_before_refundable_credits,79,79,1.0,2071.9545465783226,1.1317687601727584,,0.5443037974683544,0.5569620253164557,0.5569620253164557,0.5569620253164557,0.5537974683544303,0.6497885047161621
+gpt-5.4-nano,federal_income_tax_before_refundable_credits,84,84,1.0,2542.2835373651415,1.0465508304792916,,0.5119047619047619,0.5238095238095238,0.5238095238095238,0.5238095238095238,0.5208333333333333,0.629999094417866
gpt-5.4-nano,federal_refundable_credits,98,98,1.0,633.4683940653898,1.0,,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632
gpt-5.4-nano,free_school_meals_eligible,100,100,1.0,0.08,,0.92,0.92,0.92,0.92,0.92,0.92,0.92
-gpt-5.4-nano,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.4-nano,payroll_tax,95,95,1.0,1697.8970820055508,0.23285317448817458,,0.17894736842105263,0.23157894736842105,0.37894736842105264,0.47368421052631576,0.3157894736842105,0.634087906930071
+gpt-5.4-nano,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.4-nano,payroll_tax,96,96,1.0,1688.0404448191323,0.23447493936081706,,0.17708333333333334,0.22916666666666666,0.375,0.46875,0.3125,0.6344603342865628
gpt-5.4-nano,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.4-nano,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
gpt-5.4-nano,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
@@ -464,14 +482,14 @@ gpt-5.4-nano,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.
gpt-5.4-nano,self_employment_tax,100,100,1.0,44.312894702148434,0.37297677917394395,,0.91,0.92,0.92,0.93,0.92,0.9601618576660846
gpt-5.4-nano,snap,93,93,1.0,560.3870967741935,1.0,,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409
gpt-5.4-nano,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-gpt-5.4-nano,state_income_tax_before_refundable_credits,85,85,1.0,668.8032128670636,0.890242349338796,,0.6,0.6,0.6235294117647059,0.6470588235294118,0.6176470588235294,0.6953387957581694
-gpt-5.4-nano,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449
+gpt-5.4-nano,state_income_tax_before_refundable_credits,87,87,1.0,772.6266208462331,0.8398523345163486,,0.5862068965517241,0.5862068965517241,0.5862068965517241,0.6206896551724138,0.5948275862068965,0.6889189755482551
+gpt-5.4-nano,state_refundable_credits,98,98,1.0,123.19856564366087,1.0,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898
gpt-5.4-nano,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gpt-5.5,federal_income_tax_before_refundable_credits,79,79,1.0,371.3642350672469,0.20342378350907958,,0.7215189873417721,0.7215189873417721,0.759493670886076,0.810126582278481,0.7531645569620253,0.8861927211994403
+gpt-5.5,federal_income_tax_before_refundable_credits,84,84,1.0,342.3549977911086,0.1494399013467428,,0.7023809523809523,0.7142857142857143,0.7619047619047619,0.8095238095238095,0.7470238095238095,0.9002400422799675
gpt-5.5,federal_refundable_credits,98,98,1.0,63.1065284573302,0.1484962879061112,,0.8877551020408163,0.9081632653061225,0.9285714285714286,0.9591836734693877,0.9209183673469388,0.9629238860513549
gpt-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gpt-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.5,payroll_tax,95,95,1.0,63.22306274157075,0.007579102583782178,,0.9052631578947369,0.9157894736842105,0.9789473684210527,0.9789473684210527,0.9447368421052631,0.9952929783953353
+gpt-5.5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.5,payroll_tax,96,96,1.0,62.83667758687341,0.0076440738517140425,,0.8958333333333334,0.90625,0.9791666666666666,0.9791666666666666,0.9401041666666666,0.9952224538426787
gpt-5.5,person_chip_eligible,177,177,1.0,0.05084745762711865,,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814
gpt-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -482,14 +500,14 @@ gpt-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.9
gpt-5.5,self_employment_tax,100,100,1.0,0.018062299804685152,0.00017368058028906577,,1.0,1.0,1.0,1.0,1.0,0.9999861055535768
gpt-5.5,snap,93,93,1.0,136.08602150537635,0.3620869173659745,,0.8494623655913979,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8575268817204301,0.8956222588628208
gpt-5.5,ssi,97,97,1.0,117.52577319587628,0.0019214346712211785,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9896511044397687
-gpt-5.5,state_income_tax_before_refundable_credits,85,85,1.0,67.7357382025416,0.6061802348983927,,0.7058823529411765,0.7411764705882353,0.8588235294117647,0.9058823529411765,0.8029411764705883,0.9302852378267226
-gpt-5.5,state_refundable_credits,98,98,1.0,32.649730281635215,0.38087954249699574,,0.8571428571428571,0.8571428571428571,0.8673469387755102,0.8775510204081632,0.8647959183673469,0.9096343697454499
+gpt-5.5,state_income_tax_before_refundable_credits,87,87,1.0,85.53598176128345,0.5671116075645053,,0.6896551724137931,0.7471264367816092,0.8390804597701149,0.9080459770114943,0.7959770114942529,0.9318737033658061
+gpt-5.5,state_refundable_credits,98,98,1.0,37.76756550166073,0.34909960295309217,,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8877551020408163,0.875,0.9190337423448717
gpt-5.5,tanf,100,100,1.0,8.08958984375,0.13338243283410497,,0.99,0.99,0.99,0.99,0.99,0.9986661756716589
-gpt-5.6-luna,federal_income_tax_before_refundable_credits,79,79,1.0,731.5635824762658,0.3418311695018582,,0.810126582278481,0.8481012658227848,0.8607594936708861,0.8860759493670886,0.8512658227848101,0.9178440406522718
+gpt-5.6-luna,federal_income_tax_before_refundable_credits,84,84,1.0,727.9745624302456,0.30947344447240277,,0.7619047619047619,0.8214285714285714,0.8452380952380952,0.8690476190476191,0.8244047619047619,0.9162547793461974
gpt-5.6-luna,federal_refundable_credits,98,98,1.0,62.033617454061705,0.03964802280153711,,0.9081632653061225,0.9081632653061225,0.9387755102040817,0.9489795918367347,0.9260204081632654,0.9649374668283988
gpt-5.6-luna,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gpt-5.6-luna,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.6-luna,payroll_tax,95,95,1.0,153.78325124897202,0.059250227998888086,,0.9157894736842105,0.9368421052631579,0.9578947368421052,0.968421052631579,0.9447368421052631,0.9760668347013581
+gpt-5.6-luna,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.6-luna,payroll_tax,96,96,1.0,152.46050996398927,0.05845891868941205,,0.90625,0.9270833333333334,0.9583333333333334,0.96875,0.9401041666666667,0.9761935169499866
gpt-5.6-luna,person_chip_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542
gpt-5.6-luna,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.6-luna,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -500,14 +518,14 @@ gpt-5.6-luna,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.9
gpt-5.6-luna,self_employment_tax,100,100,1.0,34.44594216308594,0.00011566184949856606,,0.98,0.99,0.99,0.99,0.9875,0.98999074705204
gpt-5.6-luna,snap,93,93,1.0,203.15354838709675,0.7818908704351123,,0.8602150537634409,0.8817204301075269,0.8924731182795699,0.9032258064516129,0.8844086021505376,0.9019856033181286
gpt-5.6-luna,ssi,97,97,1.0,98.22680412371135,0.0022701475595913734,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9896439144833074
-gpt-5.6-luna,state_income_tax_before_refundable_credits,85,85,1.0,235.13707151538625,0.40457037523969036,,0.611764705882353,0.6352941176470588,0.7058823529411765,0.7529411764705882,0.6764705882352942,0.8080085534899788
-gpt-5.6-luna,state_refundable_credits,98,98,1.0,56.63304424149649,0.7593327685591696,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8507225538518426
+gpt-5.6-luna,state_income_tax_before_refundable_credits,87,87,1.0,250.5599217032509,0.3793700339635634,,0.6091954022988506,0.632183908045977,0.6896551724137931,0.735632183908046,0.6666666666666667,0.8121011135141236
+gpt-5.6-luna,state_refundable_credits,98,98,1.0,54.95496109417506,0.7250981251908964,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8644095100336775
gpt-5.6-luna,tanf,100,100,1.0,17.50958984375,0.28870087809100625,,0.99,0.99,0.99,0.99,0.99,0.9971129912190899
-gpt-5.6-sol,federal_income_tax_before_refundable_credits,79,79,1.0,241.22501033425638,0.13077400269025669,,0.9240506329113924,0.9367088607594937,0.9367088607594937,0.9367088607594937,0.9335443037974684,0.9516045152466146
+gpt-5.6-sol,federal_income_tax_before_refundable_credits,84,84,1.0,166.01906024460567,0.07342580484999645,,0.8809523809523809,0.9166666666666666,0.9285714285714286,0.9285714285714286,0.9136904761904763,0.96853179792143
gpt-5.6-sol,federal_refundable_credits,98,98,1.0,5.7052631412428205,0.004040567642417285,,0.9489795918367347,0.9693877551020408,0.9897959183673469,0.9897959183673469,0.9744897959183674,0.9893423852646266
gpt-5.6-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.6-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.6-sol,payroll_tax,95,95,1.0,26.13334385793587,0.010602248503188031,,0.9578947368421052,0.9578947368421052,0.9894736842105263,0.9894736842105263,0.9736842105263158,0.9934154456664409
+gpt-5.6-sol,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.6-sol,payroll_tax,96,96,1.0,26.251955774943045,0.010700216390808658,,0.9479166666666666,0.9479166666666666,0.9895833333333334,0.9895833333333334,0.96875,0.9933123647557446
gpt-5.6-sol,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514
gpt-5.6-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.6-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -518,14 +536,14 @@ gpt-5.6-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0
gpt-5.6-sol,self_employment_tax,100,100,1.0,0.0045822998046855905,2.7091941169829102e-05,,1.0,1.0,1.0,1.0,1.0,0.9999978326447063
gpt-5.6-sol,snap,93,93,1.0,106.06451612903226,0.4119675302124047,,0.8817204301075269,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.8978494623655914,0.920907764593965
gpt-5.6-sol,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.6-sol,state_income_tax_before_refundable_credits,85,85,1.0,80.82234005398918,0.15576932179888942,,0.7294117647058823,0.7411764705882353,0.8470588235294118,0.8705882352941177,0.7970588235294118,0.911561054915673
-gpt-5.6-sol,state_refundable_credits,98,98,1.0,60.30785531954862,0.4599184508904545,,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8951170192242023
+gpt-5.6-sol,state_income_tax_before_refundable_credits,87,87,1.0,85.44430359082276,0.14630047153021836,,0.7241379310344828,0.735632183908046,0.8160919540229885,0.8620689655172413,0.7844827586206895,0.9133871883053244
+gpt-5.6-sol,state_refundable_credits,98,98,1.0,65.42569053957415,0.4440178890300169,,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.902568325372344
gpt-5.6-sol,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565
-gpt-5.6-terra,federal_income_tax_before_refundable_credits,79,79,1.0,380.2110846024525,0.35807145808940594,,0.759493670886076,0.7974683544303798,0.8481012658227848,0.8734177215189873,0.8196202531645569,0.9252792199529276
+gpt-5.6-terra,federal_income_tax_before_refundable_credits,84,84,1.0,384.39513381231393,0.29444550831869254,,0.7261904761904762,0.7738095238095238,0.8214285714285714,0.8571428571428571,0.7946428571428571,0.929894186574556
gpt-5.6-terra,federal_refundable_credits,98,98,1.0,50.46908281598772,0.1548855545572883,,0.9081632653061225,0.9183673469387755,0.9489795918367347,0.9591836734693877,0.9336734693877551,0.9622067234680595
gpt-5.6-terra,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gpt-5.6-terra,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-5.6-terra,payroll_tax,95,95,1.0,47.11434155530427,0.018649035713072536,,0.8736842105263158,0.8947368421052632,0.9263157894736842,0.9473684210526315,0.9105263157894737,0.9884179672939866
+gpt-5.6-terra,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-5.6-terra,payroll_tax,96,96,1.0,46.87481624857585,0.018514792270148853,,0.8645833333333334,0.8854166666666666,0.9270833333333334,0.9479166666666666,0.90625,0.9884282548311569
gpt-5.6-terra,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.6-terra,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-5.6-terra,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -536,14 +554,14 @@ gpt-5.6-terra,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.
gpt-5.6-terra,self_employment_tax,100,100,1.0,34.42530623324218,2.7180447370748754e-05,,0.99,0.99,0.99,0.99,0.99,0.9799978255642102
gpt-5.6-terra,snap,93,93,1.0,99.8731182795699,0.47646152716612583,,0.8709677419354839,0.9032258064516129,0.9139784946236559,0.9139784946236559,0.9005376344086021,0.9226451628692512
gpt-5.6-terra,ssi,97,97,1.0,99.3917525773196,0.014579785218063085,,0.9690721649484536,0.979381443298969,0.9896907216494846,0.9896907216494846,0.981958762886598,0.9893901075212771
-gpt-5.6-terra,state_income_tax_before_refundable_credits,85,85,1.0,132.80785453872682,0.2680899698974722,,0.6823529411764706,0.7176470588235294,0.7647058823529411,0.8352941176470589,0.75,0.8658209342204997
-gpt-5.6-terra,state_refundable_credits,98,98,1.0,51.78011618575272,0.7670587174238916,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8489075825139791
+gpt-5.6-terra,state_income_tax_before_refundable_credits,87,87,1.0,153.89901758991985,0.24884824969250752,,0.6781609195402298,0.7011494252873564,0.7471264367816092,0.8390804597701149,0.7413793103448276,0.8695987677620853
+gpt-5.6-terra,state_refundable_credits,98,98,1.0,56.89795140577823,0.7533562890370616,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8591116641466322
gpt-5.6-terra,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-gpt-6-astra,federal_income_tax_before_refundable_credits,79,79,1.0,164.66130340189872,0.1202215259695243,,0.8734177215189873,0.8734177215189873,0.8860759493670886,0.8860759493670886,0.8797468354430379,0.9528244644929715
+gpt-6-astra,federal_income_tax_before_refundable_credits,84,84,1.0,255.76721923828123,0.11543878446540361,,0.8452380952380952,0.8452380952380952,0.8571428571428571,0.8690476190476191,0.8541666666666666,0.9505262352291128
gpt-6-astra,federal_refundable_credits,98,98,1.0,0.0019240149673053412,7.631848793559927e-06,,1.0,1.0,1.0,1.0,1.0,0.9999991433639108
gpt-6-astra,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-astra,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-astra,payroll_tax,95,95,1.0,85.98109696237661,0.019479632512565497,,0.8736842105263158,0.8842105263157894,0.9263157894736842,0.9368421052631579,0.9052631578947368,0.9879021229658805
+gpt-6-astra,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-6-astra,payroll_tax,96,96,1.0,85.08546145121254,0.01915497261410731,,0.875,0.8854166666666666,0.9270833333333334,0.9375,0.90625,0.9880281421161831
gpt-6-astra,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6-astra,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6-astra,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -554,14 +572,14 @@ gpt-6-astra,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0
gpt-6-astra,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
gpt-6-astra,snap,93,93,1.0,51.7632258064516,0.16769533166098569,,0.9032258064516129,0.946236559139785,0.946236559139785,0.946236559139785,0.935483870967742,0.955053340735561
gpt-6-astra,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-astra,state_income_tax_before_refundable_credits,85,85,1.0,71.37472089498183,0.12378933258050984,,0.7764705882352941,0.8235294117647058,0.8470588235294118,0.8823529411764706,0.8323529411764706,0.9460012865313553
-gpt-6-astra,state_refundable_credits,98,98,1.0,47.45212313434056,2.5319099387418778,,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8775510204081632,0.8724489795918366,0.9135804516381729
+gpt-6-astra,state_income_tax_before_refundable_credits,87,87,1.0,107.899448261261,0.13292558385438025,,0.7701149425287356,0.7931034482758621,0.8390804597701149,0.8735632183908046,0.8189655172413792,0.9411414586265999
+gpt-6-astra,state_refundable_credits,98,98,1.0,52.95771345640689,0.2787581021730102,,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8775510204081632,0.8724489795918366,0.9108276761536616
gpt-6-astra,tanf,100,100,1.0,0.01208984375,0.00019933924996272452,,0.99,1.0,1.0,1.0,0.9975,0.9999980066075004
-gpt-6-luna,federal_income_tax_before_refundable_credits,79,79,1.0,109.24773499307753,0.0944049076872314,,0.8354430379746836,0.8860759493670886,0.9113924050632911,0.9367088607594937,0.8924050632911392,0.9502968083758965
+gpt-6-luna,federal_income_tax_before_refundable_credits,84,84,1.0,103.8390073939732,0.07584305342010016,,0.7976190476190477,0.8690476190476191,0.9047619047619048,0.9285714285714286,0.875,0.9555910723437667
gpt-6-luna,federal_refundable_credits,98,98,1.0,18.44153642926897,0.05039417377182949,,0.9489795918367347,0.9693877551020408,0.9897959183673469,0.9897959183673469,0.9744897959183674,0.9943435111072436
gpt-6-luna,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-gpt-6-luna,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-luna,payroll_tax,95,95,1.0,6.8804012900904645,0.0009362983277494716,,0.9368421052631579,0.968421052631579,1.0,1.0,0.9763157894736842,0.9994185094596083
+gpt-6-luna,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-6-luna,payroll_tax,96,96,1.0,7.199564692179362,0.001195365384960741,,0.9270833333333334,0.9583333333333334,1.0,1.0,0.9713541666666667,0.9992528966343995
gpt-6-luna,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514
gpt-6-luna,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6-luna,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -572,14 +590,14 @@ gpt-6-luna,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,
gpt-6-luna,self_employment_tax,100,100,1.0,34.46423132324219,0.00014640257678745792,,0.99,0.99,0.99,0.99,0.99,0.989988287793857
gpt-6-luna,snap,93,93,1.0,104.26451612903226,0.47748056751353485,,0.8602150537634409,0.8924731182795699,0.9139784946236559,0.9139784946236559,0.8951612903225806,0.9134335298805345
gpt-6-luna,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-luna,state_income_tax_before_refundable_credits,85,85,1.0,64.46804495575849,0.08635934316176565,,0.7411764705882353,0.7529411764705882,0.8588235294117647,0.8941176470588236,0.8117647058823529,0.9234774005683388
-gpt-6-luna,state_refundable_credits,98,98,1.0,43.718861553231065,0.4563971920076426,,0.8673469387755102,0.8673469387755102,0.8877551020408163,0.8877551020408163,0.8775510204081631,0.9065499925781576
+gpt-6-luna,state_income_tax_before_refundable_credits,87,87,1.0,128.4882197708919,0.11275729762707573,,0.735632183908046,0.735632183908046,0.8275862068965517,0.8735632183908046,0.7931034482758621,0.9138451008455245
+gpt-6-luna,state_refundable_credits,98,98,1.0,48.83669677325658,0.42905946714083604,,0.8775510204081632,0.8775510204081632,0.8979591836734694,0.8979591836734694,0.8877551020408163,0.9159493651775795
gpt-6-luna,tanf,100,100,1.0,7.66958984375,0.1264574066124597,,0.99,0.99,0.99,0.99,0.99,0.9987354259338754
-gpt-6-sol,federal_income_tax_before_refundable_credits,79,79,1.0,90.57358423160598,0.06586719699147744,,0.9113924050632911,0.9367088607594937,0.9367088607594937,0.9493670886075949,0.9335443037974684,0.9741533783957494
+gpt-6-sol,federal_income_tax_before_refundable_credits,84,84,1.0,57.18018926711305,0.040452218427305904,,0.9047619047619048,0.9285714285714286,0.9404761904761905,0.9642857142857143,0.9345238095238095,0.982663334959726
gpt-6-sol,federal_refundable_credits,98,98,1.0,11.101151414520874,0.022654343243450975,,0.9489795918367347,0.9591836734693877,0.9897959183673469,0.9897959183673469,0.9719387755102041,0.9974571655543064
gpt-6-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6-sol,payroll_tax,95,95,1.0,0.006451156455601916,6.940678000524343e-06,,1.0,1.0,1.0,1.0,1.0,0.9999956894736628
+gpt-6-sol,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-6-sol,payroll_tax,96,96,1.0,0.00638487243653163,6.825643451752413e-06,,1.0,1.0,1.0,1.0,1.0,0.9999957339728427
gpt-6-sol,person_chip_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758
gpt-6-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -590,14 +608,14 @@ gpt-6-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1
gpt-6-sol,self_employment_tax,100,100,1.0,0.005250299804688438,3.994699523699791e-05,,1.0,1.0,1.0,1.0,1.0,0.9999968042403811
gpt-6-sol,snap,93,93,1.0,115.83225806451613,0.5855172715081373,,0.8924731182795699,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.9005376344086021,0.9202592344486833
gpt-6-sol,ssi,97,97,1.0,10.762886597938145,0.05572160546541417,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9988511009182389
-gpt-6-sol,state_income_tax_before_refundable_credits,85,85,1.0,75.00389639590097,0.15156732808058432,,0.7529411764705882,0.7764705882352941,0.8588235294117647,0.8705882352941177,0.8147058823529412,0.9129946763019186
-gpt-6-sol,state_refundable_credits,98,98,1.0,32.428406478920756,0.5121929777284749,,0.8571428571428571,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.8622448979591837,0.8957196571519128
+gpt-6-sol,state_income_tax_before_refundable_credits,87,87,1.0,96.68425188913457,0.12542715574026014,,0.7586206896551724,0.7701149425287356,0.8275862068965517,0.8850574712643678,0.8103448275862069,0.9208248065753097
+gpt-6-sol,state_refundable_credits,98,98,1.0,37.54624169894627,0.4834984470066206,,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8775510204081632,0.8724489795918366,0.9059237387845659
gpt-6-sol,tanf,100,100,1.0,16.45958984375,0.27138831253689305,,0.99,0.99,0.99,0.99,0.99,0.9972861168746312
-gpt-6.1-sol,federal_income_tax_before_refundable_credits,79,79,1.0,182.3848219936709,0.15969675626835553,,0.8734177215189873,0.8734177215189873,0.8734177215189873,0.8860759493670886,0.8765822784810126,0.9373341842491264
+gpt-6.1-sol,federal_income_tax_before_refundable_credits,84,84,1.0,232.1622043573289,0.13707563575845938,,0.8333333333333334,0.8333333333333334,0.8452380952380952,0.8809523809523809,0.8482142857142857,0.9412532989606603
gpt-6.1-sol,federal_refundable_credits,98,98,1.0,3.978148504763224,7.631848793559927e-06,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897950617312578
gpt-6.1-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6.1-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6.1-sol,payroll_tax,95,95,1.0,86.78852321648844,0.02630479570585291,,0.8631578947368421,0.8631578947368421,0.9052631578947369,0.9368421052631579,0.8921052631578947,0.9836633374037336
+gpt-6.1-sol,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+gpt-6.1-sol,payroll_tax,96,96,1.0,86.00312101745601,0.025949764185682987,,0.8541666666666666,0.8645833333333334,0.90625,0.9375,0.890625,0.9837813973839481
gpt-6.1-sol,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6.1-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
gpt-6.1-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -608,14 +626,14 @@ gpt-6.1-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0
gpt-6.1-sol,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453
gpt-6.1-sol,snap,93,93,1.0,64.50193548387097,0.25887264064273635,,0.9139784946236559,0.9247311827956989,0.9247311827956989,0.9247311827956989,0.9220430107526881,0.942308125501553
gpt-6.1-sol,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-gpt-6.1-sol,state_income_tax_before_refundable_credits,85,85,1.0,64.82419820426493,0.13643962898084522,,0.7529411764705882,0.7764705882352941,0.8352941176470589,0.8823529411764706,0.8117647058823529,0.941685303053594
-gpt-6.1-sol,state_refundable_credits,98,98,1.0,21.216245583320152,0.19622238526538657,,0.8979591836734694,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9056122448979591,0.9435509904614596
+gpt-6.1-sol,state_income_tax_before_refundable_credits,87,87,1.0,93.39490651602036,0.14069428442199888,,0.7471264367816092,0.7586206896551724,0.8390804597701149,0.8850574712643678,0.807471264367816,0.9383733009530807
+gpt-6.1-sol,state_refundable_credits,98,98,1.0,26.334080803345664,0.16564142833534581,,0.9081632653061225,0.9081632653061225,0.9183673469387755,0.9285714285714286,0.9158163265306123,0.9508581195744809
gpt-6.1-sol,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565
-grok-4.3,federal_income_tax_before_refundable_credits,79,79,1.0,1078.1078421677216,0.992385961685268,,0.569620253164557,0.6075949367088608,0.6329113924050633,0.6962025316455697,0.6265822784810127,0.7713660677103867
+grok-4.3,federal_income_tax_before_refundable_credits,84,84,1.0,1426.5335780552455,0.906425653849792,,0.5357142857142857,0.5714285714285714,0.6071428571428571,0.6785714285714286,0.5982142857142858,0.7668800205542471
grok-4.3,federal_refundable_credits,98,98,1.0,536.376557330696,0.8611609779771734,,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.872726829002562
grok-4.3,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-grok-4.3,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-4.3,payroll_tax,95,95,1.0,380.2779560289885,0.04816673992769779,,0.8105263157894737,0.8842105263157894,0.8947368421052632,0.9157894736842105,0.8763157894736842,0.9700859194133246
+grok-4.3,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+grok-4.3,payroll_tax,96,96,1.0,378.6777680714925,0.049023262426561666,,0.8020833333333334,0.875,0.8854166666666666,0.9166666666666666,0.8697916666666666,0.969360460983399
grok-4.3,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113
grok-4.3,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
grok-4.3,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -626,14 +644,14 @@ grok-4.3,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.
grok-4.3,self_employment_tax,100,100,1.0,34.579960327148434,0.0007042345874243298,,0.96,0.99,0.99,0.99,0.9824999999999999,0.989943661233006
grok-4.3,snap,93,93,1.0,573.2903225806451,0.9432999432999433,,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8466354917967822
grok-4.3,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-grok-4.3,state_income_tax_before_refundable_credits,85,85,1.0,198.120638404173,0.3910368271161623,,0.6588235294117647,0.6705882352941176,0.7058823529411765,0.7411764705882353,0.6941176470588235,0.8548227295721328
-grok-4.3,state_refundable_credits,98,98,1.0,112.97868960730884,0.9873021683407591,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8186587854067994
+grok-4.3,state_income_tax_before_refundable_credits,87,87,1.0,362.42858877401244,0.4003619647101831,,0.6436781609195402,0.6551724137931034,0.6896551724137931,0.735632183908046,0.6810344827586207,0.8458480355630381
+grok-4.3,state_refundable_credits,98,98,1.0,118.09652482733435,0.9865552370666861,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8288628670394524
grok-4.3,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-grok-4.5,federal_income_tax_before_refundable_credits,79,79,1.0,703.2715461827531,0.527961826191798,,0.569620253164557,0.5822784810126582,0.6075949367088608,0.6455696202531646,0.6012658227848101,0.8099980019529115
+grok-4.5,federal_income_tax_before_refundable_credits,84,84,1.0,853.7425740559896,0.4641123224199793,,0.5357142857142857,0.5476190476190477,0.5714285714285714,0.6190476190476191,0.5684523809523809,0.8137691150339345
grok-4.5,federal_refundable_credits,98,98,1.0,90.67696551887356,0.201694384459498,,0.8877551020408163,0.9081632653061225,0.9183673469387755,0.9387755102040817,0.913265306122449,0.9569526711320971
grok-4.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-grok-4.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-4.5,payroll_tax,95,95,1.0,52.45429362150494,0.01317943576331682,,0.8947368421052632,0.9263157894736842,0.9789473684210527,0.9894736842105263,0.9473684210526315,0.9918148767364665
+grok-4.5,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+grok-4.5,payroll_tax,96,96,1.0,53.76893548075358,0.014267688033320843,,0.8854166666666666,0.9166666666666666,0.96875,0.9895833333333334,0.9401041666666666,0.9910826949791746
grok-4.5,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514
grok-4.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
grok-4.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -644,14 +662,14 @@ grok-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.
grok-4.5,self_employment_tax,100,100,1.0,0.016382299804686227,0.00019320155724966045,,1.0,1.0,1.0,1.0,1.0,0.99998454387542
grok-4.5,snap,93,93,1.0,76.6236559139785,0.49252935709384804,,0.8494623655913979,0.8602150537634409,0.9139784946236559,0.9247311827956989,0.8870967741935484,0.9223451052554452
grok-4.5,ssi,97,97,1.0,11.030927835051546,0.05674460161430561,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.998830008214138
-grok-4.5,state_income_tax_before_refundable_credits,85,85,1.0,148.88173531363992,0.18809265105015857,,0.6588235294117647,0.6705882352941176,0.7058823529411765,0.7529411764705882,0.6970588235294117,0.8534742719946519
-grok-4.5,state_refundable_credits,98,98,1.0,58.54218236280947,0.7499925769866382,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8418380981044951
+grok-4.5,state_income_tax_before_refundable_credits,87,87,1.0,129.0884265187143,0.12798598674717404,,0.6436781609195402,0.6436781609195402,0.7241379310344828,0.7931034482758621,0.7011494252873564,0.8739360277107772
+grok-4.5,state_refundable_credits,98,98,1.0,63.66001758283498,0.7352862579858523,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.852042179737148
grok-4.5,tanf,100,100,1.0,16.44958984375,0.2712234309601872,,0.99,0.99,0.99,0.99,0.99,0.9972877656903981
-grok-4.6,federal_income_tax_before_refundable_credits,79,79,1.0,783.8641836431962,0.5313118070569347,,0.5822784810126582,0.5822784810126582,0.6329113924050633,0.6582278481012658,0.6139240506329113,0.8317306011629841
+grok-4.6,federal_income_tax_before_refundable_credits,84,84,1.0,851.6149931408111,0.43307767621650683,,0.5476190476190477,0.5476190476190477,0.6071428571428571,0.6309523809523809,0.5833333333333334,0.8395558363575129
grok-4.6,federal_refundable_credits,98,98,1.0,61.75859817193479,0.11442975660933365,,0.9183673469387755,0.9285714285714286,0.9489795918367347,0.9693877551020408,0.9413265306122449,0.987155843645891
grok-4.6,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-grok-4.6,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-4.6,payroll_tax,95,95,1.0,18.936030299136515,0.0016059371922763725,,0.9157894736842105,0.9578947368421052,1.0,1.0,0.968421052631579,0.9990026284805864
+grok-4.6,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+grok-4.6,payroll_tax,96,96,1.0,19.24565406799317,0.0019353945334378885,,0.90625,0.9479166666666666,1.0,1.0,0.9635416666666666,0.9987903784166013
grok-4.6,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757
grok-4.6,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
grok-4.6,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -662,14 +680,14 @@ grok-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.
grok-4.6,self_employment_tax,100,100,1.0,0.015082299804685135,0.00018936798364602316,,1.0,1.0,1.0,1.0,1.0,0.9999848505613084
grok-4.6,snap,93,93,1.0,89.39784946236558,0.4142431666072162,,0.8279569892473119,0.8387096774193549,0.8924731182795699,0.8924731182795699,0.8629032258064516,0.9098369767108193
grok-4.6,ssi,97,97,1.0,3.9587628865979383,0.020130596490479824,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9995849361548356
-grok-4.6,state_income_tax_before_refundable_credits,85,85,1.0,105.6576616118936,0.1726877719717267,,0.6705882352941176,0.7176470588235294,0.7764705882352941,0.8235294117647058,0.7470588235294118,0.8940241719155285
-grok-4.6,state_refundable_credits,98,98,1.0,44.256468077095185,0.49418623535212247,,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.909231099629202
+grok-4.6,state_income_tax_before_refundable_credits,87,87,1.0,98.79266698881128,0.13243909780341134,,0.6551724137931034,0.7241379310344828,0.7816091954022989,0.8390804597701149,0.75,0.9068320456102785
+grok-4.6,state_refundable_credits,98,98,1.0,49.3743032971207,0.47293715446914036,,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.9179598813675981
grok-4.6,tanf,100,100,1.0,16.48958984375,0.27188295726701056,,0.99,0.99,0.99,0.99,0.99,0.9972811704273299
-grok-4.7,federal_income_tax_before_refundable_credits,79,79,1.0,474.8654557703718,0.37521192738038545,,0.6075949367088608,0.7215189873417721,0.8227848101265823,0.8227848101265823,0.7436708860759493,0.9038921815810197
+grok-4.7,federal_income_tax_before_refundable_credits,84,84,1.0,477.46425688244045,0.3066532799688514,,0.5714285714285714,0.6904761904761905,0.7976190476190477,0.8095238095238095,0.7172619047619047,0.9059521828039663
grok-4.7,federal_refundable_credits,98,98,1.0,28.61213208256935,0.03835701888150592,,0.9081632653061225,0.9489795918367347,0.9489795918367347,0.9897959183673469,0.9489795918367347,0.9956946203296269
grok-4.7,free_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97
-grok-4.7,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-4.7,payroll_tax,95,95,1.0,3.9950602847450707,0.0008883315813322884,,0.9473684210526315,0.968421052631579,1.0,1.0,0.9789473684210526,0.999448299333699
+grok-4.7,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+grok-4.7,payroll_tax,96,96,1.0,4.344279322306318,0.0011481980843171774,,0.9375,0.9583333333333334,1.0,1.0,0.9739583333333334,0.9992823761973018
grok-4.7,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757
grok-4.7,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
grok-4.7,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -680,14 +698,14 @@ grok-4.7,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.
grok-4.7,self_employment_tax,100,100,1.0,0.013281323242185863,9.278463705377946e-05,,1.0,1.0,1.0,1.0,1.0,0.9999925772290357
grok-4.7,snap,93,93,1.0,81.92580645161291,0.33514802771074387,,0.8602150537634409,0.8709677419354839,0.9247311827956989,0.9247311827956989,0.8951612903225806,0.9316459746210789
grok-4.7,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-4.7,state_income_tax_before_refundable_credits,85,85,1.0,83.76819463119507,0.14155778582620437,,0.7058823529411765,0.7411764705882353,0.788235294117647,0.8470588235294118,0.7705882352941177,0.9046449907181183
-grok-4.7,state_refundable_credits,98,98,1.0,27.02574830755896,0.3738464532377186,,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8673469387755102,0.921130243282868
+grok-4.7,state_income_tax_before_refundable_credits,87,87,1.0,76.4456632846087,0.11228770402555928,,0.7011494252873564,0.7471264367816092,0.7701149425287356,0.8620689655172413,0.7701149425287357,0.9140124273012373
+grok-4.7,state_refundable_credits,98,98,1.0,29.83746107860488,0.347235601631734,,0.8673469387755102,0.8775510204081632,0.8775510204081632,0.8877551020408163,0.8775510204081631,0.9295611711455155
grok-4.7,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565
-grok-build-0.1,federal_income_tax_before_refundable_credits,79,79,1.0,936.1758708712421,0.436166142115378,,0.4936708860759494,0.4936708860759494,0.5443037974683544,0.5569620253164557,0.5221518987341772,0.7399136659076458
+grok-build-0.1,federal_income_tax_before_refundable_credits,84,84,1.0,1013.741460890997,0.3734958875522378,,0.4642857142857143,0.4642857142857143,0.5357142857142857,0.5476190476190477,0.5029761904761905,0.7517608647715179
grok-build-0.1,federal_refundable_credits,98,98,1.0,111.45127993213887,0.20595095265447802,,0.8469387755102041,0.8469387755102041,0.8673469387755102,0.8979591836734694,0.864795918367347,0.9156585665387831
grok-build-0.1,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-grok-build-0.1,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-grok-build-0.1,payroll_tax,95,95,1.0,185.19966120990955,0.04169540670884422,,0.7473684210526316,0.8,0.8421052631578947,0.8842105263157894,0.8184210526315789,0.9741049579387179
+grok-build-0.1,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+grok-build-0.1,payroll_tax,96,96,1.0,185.63153882344565,0.04265978476135565,,0.7395833333333334,0.7916666666666666,0.8333333333333334,0.8854166666666666,0.8125,0.9733376345241527
grok-build-0.1,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028
grok-build-0.1,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
grok-build-0.1,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -698,14 +716,14 @@ grok-build-0.1,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0
grok-build-0.1,self_employment_tax,100,100,1.0,19.800231323242187,0.00021513468896632062,,0.99,0.99,0.99,0.99,0.99,0.9899827892248827
grok-build-0.1,snap,93,93,1.0,253.53763440860214,0.5006941553309233,,0.8387096774193549,0.8602150537634409,0.8709677419354839,0.8817204301075269,0.8629032258064516,0.9085051180720215
grok-build-0.1,ssi,97,97,1.0,106.9278350515464,0.04653972128171252,,0.9690721649484536,0.9690721649484536,0.9896907216494846,0.9896907216494846,0.9793814432989691,0.9887311397673872
-grok-build-0.1,state_income_tax_before_refundable_credits,85,85,1.0,222.71092340244965,0.7188498118880351,,0.5411764705882353,0.5647058823529412,0.611764705882353,0.6470588235294118,0.5911764705882353,0.7586843042683711
-grok-build-0.1,state_refundable_credits,98,98,1.0,75.52452420701786,1.2333790047333604,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8271666919340446
+grok-build-0.1,state_income_tax_before_refundable_credits,87,87,1.0,211.87875777825542,0.6318095813983571,,0.5172413793103449,0.5402298850574713,0.632183908045977,0.6896551724137931,0.5948275862068966,0.7787208434967298
+grok-build-0.1,state_refundable_credits,98,98,1.0,81.03011452908419,0.8786861288508024,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8271666919340446
grok-build-0.1,tanf,100,100,1.0,6.00958984375,0.09908706487929027,,0.99,0.99,0.99,1.0,0.9924999999999999,0.9990091293512071
-inkling,federal_income_tax_before_refundable_credits,79,79,1.0,289.91852390723886,0.18090874664739845,,0.759493670886076,0.7974683544303798,0.7974683544303798,0.810126582278481,0.7911392405063291,0.9290104918219068
+inkling,federal_income_tax_before_refundable_credits,84,84,1.0,263.80070452008925,0.1432112142401804,,0.7142857142857143,0.7738095238095238,0.7976190476190477,0.8214285714285714,0.7767857142857142,0.9386237653256371
inkling,federal_refundable_credits,98,98,1.0,81.34820276377152,0.10642112209169272,,0.8877551020408163,0.9285714285714286,0.9285714285714286,0.9489795918367347,0.9234693877551021,0.9472384454795039
inkling,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-inkling,local_income_tax,100,100,1.0,39.24,,,0.99,0.99,0.99,0.99,0.99,0.99
-inkling,payroll_tax,95,95,1.0,85.36980892783718,0.020265822432288336,,0.8421052631578947,0.8736842105263158,0.9157894736842105,0.9578947368421052,0.8973684210526316,0.9874138576473158
+inkling,local_income_tax,98,98,1.0,40.04081632653061,,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469
+inkling,payroll_tax,96,96,1.0,85.51866416931153,0.02065763564885749,,0.8333333333333334,0.8645833333333334,0.9166666666666666,0.9583333333333334,0.8932291666666667,0.9870889777194641
inkling,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
inkling,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
inkling,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -716,14 +734,14 @@ inkling,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.9
inkling,self_employment_tax,100,100,1.0,34.43888229980468,0.0001841476840988503,,0.99,0.99,0.99,0.99,0.99,0.9899852681852721
inkling,snap,93,93,1.0,149.09677419354838,0.4805822155985559,,0.8602150537634409,0.9032258064516129,0.9247311827956989,0.9247311827956989,0.9032258064516129,0.9238612673536069
inkling,ssi,97,97,1.0,10.68041237113402,0.05529461998292058,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9988599047426203
-inkling,state_income_tax_before_refundable_credits,85,85,1.0,132.78373359029433,0.24620312430719224,,0.611764705882353,0.6470588235294118,0.7294117647058823,0.7764705882352941,0.6911764705882353,0.8437998493022588
-inkling,state_refundable_credits,98,98,1.0,62.31431975170057,0.7678618120653087,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8316326530612246,0.8707272557191053
+inkling,state_income_tax_before_refundable_credits,87,87,1.0,193.81175895405914,0.2232294577668898,,0.5977011494252874,0.632183908045977,0.7126436781609196,0.7701149425287356,0.67816091954023,0.8499168345382411
+inkling,state_refundable_credits,98,98,1.0,67.43215497172609,0.7542066245397384,,0.8367346938775511,0.8367346938775511,0.8469387755102041,0.8469387755102041,0.8418367346938777,0.8809313373517583
inkling,tanf,100,100,1.0,16.44958984375,0.2712234309601872,,0.99,0.99,0.99,0.99,0.99,0.9972877656903981
-kimi-k2.6,federal_income_tax_before_refundable_credits,79,45,0.569620253164557,3575.6949115668403,0.12506863265036838,,0.46835443037974683,0.46835443037974683,0.5189873417721519,0.5189873417721519,0.49367088607594933,0.5458799945752838
+kimi-k2.6,federal_income_tax_before_refundable_credits,84,48,0.5714285714285714,3638.7202458699544,0.19976707231664237,,0.44047619047619047,0.44047619047619047,0.488095238095238,0.488095238095238,0.4642857142857143,0.5357420152003997
kimi-k2.6,federal_refundable_credits,98,48,0.4897959183673469,174.12708188374836,0.36096984340474547,,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.4648583737385818
kimi-k2.6,free_school_meals_eligible,100,93,0.93,0.021505376344086023,,0.91,0.91,0.91,0.91,0.91,0.91,0.91
-kimi-k2.6,local_income_tax,100,87,0.87,0.0,,,0.87,0.87,0.87,0.87,0.87,0.87
-kimi-k2.6,payroll_tax,95,47,0.49473684210526314,13.121414872839113,0.003717481199433475,,0.4736842105263158,0.4736842105263158,0.4842105263157894,0.49473684210526314,0.48157894736842105,0.49395421448432963
+kimi-k2.6,local_income_tax,98,85,0.8673469387755102,0.0,,,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102
+kimi-k2.6,payroll_tax,96,47,0.4895833333333333,13.121414872839113,0.003717481199433475,,0.46875,0.46875,0.47916666666666663,0.4895833333333333,0.47656249999999994,0.4888088580834512
kimi-k2.6,person_chip_eligible,177,151,0.8531073446327684,0.019867549668874173,,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955
kimi-k2.6,person_early_head_start_eligible,38,37,0.9736842105263158,0.02702702702702703,,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316
kimi-k2.6,person_head_start_eligible,38,37,0.9736842105263158,0.0,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
@@ -734,14 +752,14 @@ kimi-k2.6,reduced_price_school_meals_eligible,100,92,0.92,0.010869565217391304,,
kimi-k2.6,self_employment_tax,100,60,0.6,0.0023014322916651508,1.6540086871946673e-05,,0.6,0.6,0.6,0.6,0.6,0.5999996691982625
kimi-k2.6,snap,93,83,0.8924731182795699,139.51807228915663,0.7636339837034091,,0.8064516129032259,0.8064516129032259,0.8172043010752688,0.8279569892473118,0.814516129032258,0.826784173444868
kimi-k2.6,ssi,97,86,0.8865979381443299,0.4186046511627907,0.003842869342442357,,0.8762886597938144,0.8865979381443299,0.8865979381443299,0.8865979381443299,0.884020618556701,0.8865583209346138
-kimi-k2.6,state_income_tax_before_refundable_credits,85,57,0.6705882352941176,115.05470420134694,0.5209272704784943,,0.5294117647058824,0.5294117647058824,0.5411764705882353,0.588235294117647,0.5470588235294118,0.5911823207388108
-kimi-k2.6,state_refundable_credits,98,56,0.5714285714285714,10.34587128673281,1.0,,0.5,0.5,0.5,0.5,0.5,0.5
+kimi-k2.6,state_income_tax_before_refundable_credits,87,55,0.632183908045977,105.98538023480502,0.623656804074475,,0.5172413793103449,0.5172413793103449,0.5172413793103449,0.5632183908045977,0.5287356321839081,0.5571323624711272
+kimi-k2.6,state_refundable_credits,98,56,0.5714285714285714,10.006585572447095,1.0,,0.5102040816326531,0.5102040816326531,0.5102040816326531,0.5102040816326531,0.5102040816326531,0.5102040816326531
kimi-k2.6,tanf,100,89,0.89,0.0,,,0.89,0.89,0.89,0.89,0.89,0.89
-kimi-k3,federal_income_tax_before_refundable_credits,79,78,0.9873417721518988,207.04181865985575,0.08858167701876732,,0.8734177215189873,0.8987341772151899,0.9240506329113924,0.9240506329113924,0.9050632911392406,0.9539178322245967
+kimi-k3,federal_income_tax_before_refundable_credits,84,82,0.9761904761904762,65.79315822694355,0.027719020540418766,,0.8571428571428571,0.8928571428571429,0.9285714285714285,0.9404761904761905,0.9047619047619047,0.9649708726384021
kimi-k3,federal_refundable_credits,98,95,0.9693877551020408,17.939430497018922,0.06333102758780991,,0.9081632653061225,0.9285714285714285,0.9489795918367347,0.9489795918367347,0.9336734693877551,0.9520750887401437
kimi-k3,free_school_meals_eligible,100,97,0.97,0.010309278350515464,,0.96,0.96,0.96,0.96,0.96,0.96,0.96
-kimi-k3,local_income_tax,100,97,0.97,0.0,,,0.97,0.97,0.97,0.97,0.97,0.97
-kimi-k3,payroll_tax,95,92,0.968421052631579,5.707645502505112,0.0005809413791082061,,0.9368421052631579,0.9473684210526316,0.968421052631579,0.968421052631579,0.9552631578947368,0.9680786029765258
+kimi-k3,local_income_tax,98,95,0.9693877551020408,0.0,,,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408,0.9693877551020408
+kimi-k3,payroll_tax,96,93,0.96875,5.984876065818231,0.000813410913242172,,0.9270833333333334,0.9375,0.96875,0.96875,0.9505208333333334,0.9682670372702624
kimi-k3,person_chip_eligible,177,171,0.9661016949152542,0.029239766081871343,,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327
kimi-k3,person_early_head_start_eligible,38,36,0.9473684210526315,0.0,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
kimi-k3,person_head_start_eligible,38,36,0.9473684210526315,0.0,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
@@ -752,14 +770,14 @@ kimi-k3,reduced_price_school_meals_eligible,100,97,0.97,0.010309278350515464,,0.
kimi-k3,self_employment_tax,100,97,0.97,0.0033838142316346756,5.9567983600926226e-05,,0.97,0.97,0.97,0.97,0.97,0.969995234561312
kimi-k3,snap,93,91,0.978494623655914,85.72285714285714,0.47465267116100845,,0.8602150537634409,0.903225806451613,0.903225806451613,0.903225806451613,0.89247311827957,0.9031032923010517
kimi-k3,ssi,97,94,0.9690721649484536,11.106382978723405,0.05572160546541417,,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9679232658666925
-kimi-k3,state_income_tax_before_refundable_credits,85,84,0.9882352941176471,95.7316657054992,0.12161053618529549,,0.7058823529411765,0.7529411764705882,0.8470588235294118,0.8470588235294118,0.7882352941176471,0.9128812351389616
-kimi-k3,state_refundable_credits,98,95,0.9693877551020408,30.48218292336715,0.5969869954367941,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8163265306122448,0.8086734693877551,0.8556247048732093
+kimi-k3,state_income_tax_before_refundable_credits,87,85,0.9770114942528736,88.95859383414769,0.10241190382273486,,0.7011494252873564,0.7471264367816092,0.8275862068965517,0.8390804597701149,0.7787356321839081,0.908391434357939
+kimi-k3,state_refundable_credits,98,95,0.9693877551020408,35.7616339924461,0.5808348945039176,,0.8163265306122448,0.8163265306122448,0.8163265306122448,0.8265306122448979,0.818877551020408,0.8643534866116053
kimi-k3,tanf,100,97,0.97,16.960917364690722,0.2712646513543637,,0.96,0.96,0.96,0.96,0.96,0.9672873534864564
-minimax-m3,federal_income_tax_before_refundable_credits,79,79,1.0,1616.0155662826344,1.24797104806946,,0.5443037974683544,0.569620253164557,0.6329113924050633,0.6329113924050633,0.5949367088607596,0.6623077104735428
+minimax-m3,federal_income_tax_before_refundable_credits,84,84,1.0,2306.185767764137,1.1768839640289745,,0.5119047619047619,0.5357142857142857,0.5952380952380952,0.6071428571428571,0.5625,0.6385903441966674
minimax-m3,federal_refundable_credits,98,98,1.0,426.5208842997648,0.690454400381062,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.840867363222534
minimax-m3,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-minimax-m3,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-minimax-m3,payroll_tax,95,95,1.0,1887.1655739874589,0.24288781312557778,,0.6631578947368421,0.7263157894736842,0.8210526315789474,0.8210526315789474,0.7578947368421052,0.8491538844799043
+minimax-m3,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+minimax-m3,payroll_tax,96,96,1.0,1869.8686400095621,0.240498984404477,,0.65625,0.71875,0.8125,0.8229166666666666,0.7526041666666666,0.849688134747202
minimax-m3,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028
minimax-m3,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
minimax-m3,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315
@@ -770,14 +788,14 @@ minimax-m3,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,
minimax-m3,self_employment_tax,100,100,1.0,52.137589819335936,0.2117598110686142,,0.93,0.95,0.96,0.96,0.95,0.9630592151145109
minimax-m3,snap,93,93,1.0,426.06451612903226,0.8724417371025018,,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8709677419354839,0.8629032258064516,0.8780457786845964
minimax-m3,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969
-minimax-m3,state_income_tax_before_refundable_credits,85,85,1.0,564.1329190366408,1.3637327619424762,,0.611764705882353,0.6235294117647059,0.6588235294117647,0.6588235294117647,0.638235294117647,0.7320888435697148
-minimax-m3,state_refundable_credits,98,98,1.0,114.76440389302311,0.9506409112855011,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.825392485682255
+minimax-m3,state_income_tax_before_refundable_credits,87,87,1.0,1206.3497134625227,1.3327421607160284,,0.5977011494252874,0.6091954022988506,0.6436781609195402,0.6436781609195402,0.6235632183908046,0.7179401703167895
+minimax-m3,state_refundable_credits,98,98,1.0,119.88223911304863,0.9477374354787659,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8355965673149081
minimax-m3,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-ox-alpha,federal_income_tax_before_refundable_credits,79,79,1.0,381.06412393690664,0.27801344121959043,,0.7721518987341772,0.7848101265822784,0.810126582278481,0.8227848101265823,0.7974683544303797,0.8784885865686891
+ox-alpha,federal_income_tax_before_refundable_credits,84,84,1.0,336.8027373976934,0.21847981866323207,,0.75,0.7619047619047619,0.8095238095238095,0.8214285714285714,0.7857142857142856,0.8946873993435402
ox-alpha,federal_refundable_credits,98,98,1.0,109.45626254334742,0.08470054563216628,,0.8571428571428571,0.8979591836734694,0.8979591836734694,0.9183673469387755,0.8928571428571428,0.9394723877351651
ox-alpha,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-ox-alpha,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-ox-alpha,payroll_tax,95,95,1.0,87.63226611328125,0.006936513753927001,,0.9157894736842105,0.9368421052631579,0.968421052631579,0.9789473684210527,0.95,0.9956920598791402
+ox-alpha,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+ox-alpha,payroll_tax,96,96,1.0,89.07838742574056,0.008478742555833167,,0.90625,0.9270833333333334,0.9583333333333334,0.9791666666666666,0.9427083333333334,0.9947007859026042
ox-alpha,person_chip_eligible,177,177,1.0,0.07344632768361582,,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842
ox-alpha,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
ox-alpha,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
@@ -788,14 +806,14 @@ ox-alpha,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.
ox-alpha,self_employment_tax,100,100,1.0,23.392942163085937,0.00010208031325015381,,0.99,0.99,0.99,0.99,0.99,0.9799918335749399
ox-alpha,snap,93,93,1.0,71.8537634408602,0.4892323148995972,,0.8709677419354839,0.9139784946236559,0.9139784946236559,0.9139784946236559,0.9032258064516129,0.9224348876641922
ox-alpha,ssi,97,97,1.0,96.5,0.0,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846
-ox-alpha,state_income_tax_before_refundable_credits,85,85,1.0,87.40754708121806,0.1634743976858346,,0.6470588235294118,0.6823529411764706,0.788235294117647,0.8588235294117647,0.7441176470588236,0.8971675584365976
-ox-alpha,state_refundable_credits,98,98,1.0,45.65407537246238,0.717552980967496,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8367346938775511,0.8214285714285714,0.8580004728835212
+ox-alpha,state_income_tax_before_refundable_credits,87,87,1.0,81.96781752816564,0.12882377126148564,,0.6436781609195402,0.6781609195402298,0.7816091954022989,0.8850574712643678,0.7471264367816092,0.9081202654125741
+ox-alpha,state_refundable_credits,98,98,1.0,50.77191059248789,0.7009384504361722,,0.826530612244898,0.826530612244898,0.826530612244898,0.8469387755102041,0.8316326530612246,0.8682045545161743
ox-alpha,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-qwen-3.7-max,federal_income_tax_before_refundable_credits,79,79,1.0,1087.3418422418906,0.5540419398273054,,0.46835443037974683,0.4810126582278481,0.5569620253164557,0.5949367088607594,0.5253164556962026,0.7001634330728224
+qwen-3.7-max,federal_income_tax_before_refundable_credits,84,84,1.0,2017.7925074404761,0.6343843931737613,,0.44047619047619047,0.4523809523809524,0.5238095238095238,0.5595238095238095,0.494047619047619,0.6696556243651495
qwen-3.7-max,federal_refundable_credits,98,98,1.0,369.4145165392817,0.40785543409849506,,0.7755102040816326,0.7857142857142857,0.7857142857142857,0.7959183673469388,0.7857142857142856,0.8317713288256792
qwen-3.7-max,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99
-qwen-3.7-max,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-qwen-3.7-max,payroll_tax,95,95,1.0,336.5307187243009,0.04194101630112751,,0.7684210526315789,0.8421052631578947,0.8947368421052632,0.9157894736842105,0.8552631578947367,0.9739524214550894
+qwen-3.7-max,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+qwen-3.7-max,payroll_tax,96,96,1.0,334.31633532206223,0.04214939526213724,,0.7604166666666666,0.8333333333333334,0.8854166666666666,0.9166666666666666,0.8489583333333333,0.973656627961164
qwen-3.7-max,person_chip_eligible,177,177,1.0,0.096045197740113,,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887
qwen-3.7-max,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158
qwen-3.7-max,person_head_start_eligible,38,38,1.0,0.10526315789473684,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632
@@ -806,14 +824,14 @@ qwen-3.7-max,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.9
qwen-3.7-max,self_employment_tax,100,100,1.0,96.83744475097657,0.2059975897273345,,0.96,0.96,0.96,0.96,0.96,0.9635201928218131
qwen-3.7-max,snap,93,93,1.0,414.3333333333333,1.2063119918995897,,0.8172043010752689,0.8172043010752689,0.8387096774193549,0.8387096774193549,0.8279569892473119,0.8679149704672419
qwen-3.7-max,ssi,97,97,1.0,161.4020618556701,0.5232690124858116,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.979381443298969,0.9742268041237113,0.9789016698456533
-qwen-3.7-max,state_income_tax_before_refundable_credits,85,85,1.0,452.5277550172245,0.4580332992754896,,0.5411764705882353,0.5411764705882353,0.5529411764705883,0.5647058823529412,0.55,0.7306342680444039
-qwen-3.7-max,state_refundable_credits,98,98,1.0,98.95411766305261,0.9676512169222556,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8250067738900787
+qwen-3.7-max,state_income_tax_before_refundable_credits,87,87,1.0,516.4208806651762,0.3840996528646306,,0.5287356321839081,0.5287356321839081,0.5402298850574713,0.5632183908045977,0.5402298850574713,0.737820888390778
+qwen-3.7-max,state_refundable_credits,98,98,1.0,104.07195288307813,0.9657483473294471,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8352108555227318
qwen-3.7-max,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99
-qwen3.8-max,federal_income_tax_before_refundable_credits,79,79,1.0,1711.799140130538,1.391391758761768,,0.5189873417721519,0.5189873417721519,0.5443037974683544,0.5569620253164557,0.5348101265822784,0.6556038865143431
+qwen3.8-max,federal_income_tax_before_refundable_credits,84,84,1.0,2060.7147875976566,1.2939030717185567,,0.4880952380952381,0.4880952380952381,0.5238095238095238,0.5476190476190477,0.5119047619047619,0.65540419242608
qwen3.8-max,federal_refundable_credits,98,98,1.0,767.3067765590123,0.6106816981178466,,0.673469387755102,0.673469387755102,0.6836734693877551,0.6836734693877551,0.6785714285714286,0.7185520154782343
qwen3.8-max,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98
-qwen3.8-max,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
-qwen3.8-max,payroll_tax,95,95,1.0,550.8215041889391,0.12071392001673044,,0.7473684210526316,0.7789473684210526,0.8210526315789474,0.8421052631578947,0.7973684210526315,0.927965044861761
+qwen3.8-max,local_income_tax,98,98,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0
+qwen3.8-max,payroll_tax,96,96,1.0,547.4427376047771,0.12035985871425653,,0.7395833333333334,0.7708333333333334,0.8125,0.84375,0.7916666666666667,0.92767926062499
qwen3.8-max,person_chip_eligible,177,177,1.0,0.07909604519774012,,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598
qwen3.8-max,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0
qwen3.8-max,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579
@@ -824,6 +842,6 @@ qwen3.8-max,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0
qwen3.8-max,self_employment_tax,100,100,1.0,85.60206276855467,0.43511756292674586,,0.92,0.92,0.92,0.94,0.925,0.9551905949658603
qwen3.8-max,snap,93,93,1.0,662.9677419354839,2.919012327941218,,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8428620760225537
qwen3.8-max,ssi,97,97,1.0,285.1505154639175,0.8968785471055618,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9711983804720503
-qwen3.8-max,state_income_tax_before_refundable_credits,85,85,1.0,345.12212231961416,0.4766805271117537,,0.6235294117647059,0.6470588235294118,0.6823529411764706,0.7176470588235294,0.6676470588235294,0.8080689740052397
-qwen3.8-max,state_refundable_credits,98,98,1.0,94.59093450526802,0.9250063908137852,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8301008669933864
+qwen3.8-max,state_income_tax_before_refundable_credits,87,87,1.0,530.2710594771374,0.5012687923278506,,0.6091954022988506,0.632183908045977,0.6781609195402298,0.7126436781609196,0.6580459770114943,0.7928302507262537
+qwen3.8-max,state_refundable_credits,98,98,1.0,99.70876972529354,0.9326948279623068,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8382059992310285
qwen3.8-max,tanf,100,100,1.0,93.16958984375,1.0,,0.98,0.98,0.98,0.98,0.98,0.98
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md
index 2a8a2273..9644b3b5 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md
@@ -4,21 +4,22 @@ This report is generated by `policybench analyze` from no-tools benchmark result
## Headline
-Top model: `gpt-6-sol` with `bounded_score=0.981` and `within_1pct=0.965`.
+Top model: `gpt-6-sol` with `bounded_score=0.982` and `within_1pct=0.962`.
## Usage
-Recorded-usage cost subtotal (43 of 46 models): $450.639.
+Recorded-usage cost subtotal (44 of 47 models): $450.769.
-Published model costs total $514.719 (includes release-metadata costs for: claude-fable-5, gemini-3.6-flash, grok-build-0.1).
+Published model costs total $514.849 (includes release-metadata costs for: claude-fable-5, gemini-3.6-flash, grok-build-0.1).
-Cumulative request-time: `11326.1 min` (summed across concurrent calls; not wall-clock). Run wall-clock: `129046.0 min`.
+Cumulative request-time: `11335.3 min` (summed across concurrent calls; not wall-clock). Run wall-clock: `141956.3 min`.
| model | total_cost_usd | cost_rows_estimated | cumulative_request_time | wall_clock | total_tokens | reasoning_tokens | parsed_rows | total_rows |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| claude-fable-5 | n/a | 0 | n/a | n/a | n/a | n/a | 1984 | 1984 |
| claude-fable-5.1 | 25.698 | 0 | 93.6 min | 23.3 min | 680865.000 | 0.000 | 1984 | 1984 |
| claude-haiku-4.5 | 5.541 | 1984 | 98.8 min | n/a | 3836025.000 | 0.000 | 1984 | 1984 |
+| claude-haiku-5.5 | 0.130 | 0 | 9.2 min | 5.4 min | 759194.000 | 0.000 | 1984 | 1984 |
| claude-opus-4.7 | 29.303 | 1984 | 103.4 min | n/a | 4569140.000 | 0.000 | 1984 | 1984 |
| claude-opus-4.8 | 25.506 | 1984 | 105.7 min | n/a | 3782202.000 | 0.000 | 1984 | 1984 |
| claude-opus-5 | 6.749 | 1984 | 30.1 min | 8.6 min | 746677.000 | 0.000 | 1984 | 1984 |
@@ -67,52 +68,53 @@ Cumulative request-time: `11326.1 min` (summed across concurrent calls; not wall
| model | mean_score | mean_exact | mean_within_1pct | mean_within_5pct | mean_within_10pct | mean_binary_accuracy | mean_mae | total_n |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
-| gpt-6-sol | 0.982 | 0.962 | 0.966 | 0.973 | 0.974 | 0.996 | 19.567 | 1920 |
-| claude-opus-5.5 | 0.982 | 0.963 | 0.969 | 0.973 | 0.976 | 0.988 | 24.878 | 1920 |
-| gpt-6-luna | 0.977 | 0.949 | 0.957 | 0.970 | 0.973 | 0.989 | 21.625 | 1920 |
-| claude-sonnet-5.5 | 0.971 | 0.946 | 0.951 | 0.961 | 0.963 | 0.984 | 26.213 | 1920 |
-| gpt-6-astra | 0.981 | 0.955 | 0.961 | 0.966 | 0.969 | 0.988 | 23.408 | 1920 |
-| gpt-5.6-sol | 0.977 | 0.957 | 0.961 | 0.969 | 0.971 | 0.991 | 29.822 | 1920 |
-| grok-4.7 | 0.974 | 0.931 | 0.943 | 0.956 | 0.963 | 0.984 | 39.821 | 1920 |
-| claude-fable-5.1 | 0.978 | 0.952 | 0.960 | 0.968 | 0.971 | 0.985 | 37.642 | 1920 |
-| inkling | 0.968 | 0.927 | 0.938 | 0.947 | 0.953 | 0.993 | 50.094 | 1920 |
-| gemini-3.8-flash | 0.972 | 0.925 | 0.930 | 0.951 | 0.957 | 0.990 | 56.120 | 1920 |
-| gpt-6.1-sol | 0.979 | 0.954 | 0.956 | 0.962 | 0.967 | 0.985 | 24.459 | 1920 |
-| gemini-3.6-flash | 0.968 | 0.911 | 0.916 | 0.936 | 0.943 | 0.991 | 59.644 | 1920 |
-| gpt-5.6-terra | 0.964 | 0.929 | 0.938 | 0.948 | 0.955 | 0.985 | 53.158 | 1920 |
-| gpt-5.5 | 0.965 | 0.928 | 0.932 | 0.946 | 0.954 | 0.975 | 47.778 | 1920 |
-| deepseek-v4.1-flash | 0.966 | 0.924 | 0.932 | 0.947 | 0.954 | 0.982 | 44.587 | 1920 |
-| grok-4.6 | 0.968 | 0.924 | 0.931 | 0.944 | 0.949 | 0.986 | 62.469 | 1920 |
-| ox-alpha | 0.960 | 0.927 | 0.935 | 0.944 | 0.952 | 0.979 | 53.543 | 1920 |
-| kimi-k3 | 0.945 | 0.919 | 0.927 | 0.936 | 0.937 | 0.950 | 26.156 | 1920 |
-| gpt-5.6-luna | 0.958 | 0.929 | 0.936 | 0.944 | 0.950 | 0.982 | 88.479 | 1920 |
-| grok-4.5 | 0.960 | 0.919 | 0.924 | 0.934 | 0.941 | 0.988 | 64.336 | 1920 |
-| deepseek-v4-pro-0813 | 0.962 | 0.915 | 0.922 | 0.933 | 0.940 | 0.993 | 80.243 | 1920 |
-| gemini-3.7-flash | 0.960 | 0.912 | 0.916 | 0.931 | 0.937 | 0.989 | 81.970 | 1920 |
-| deepseek-v4-flash-0731 | 0.949 | 0.904 | 0.911 | 0.929 | 0.932 | 0.988 | 102.568 | 1920 |
-| claude-opus-5 | 0.955 | 0.911 | 0.915 | 0.926 | 0.936 | 0.982 | 86.760 | 1920 |
-| gemini-3.1-pro-preview | 0.955 | 0.903 | 0.907 | 0.918 | 0.926 | 0.985 | 83.412 | 1920 |
-| claude-fable-5 | 0.955 | 0.911 | 0.915 | 0.928 | 0.935 | 0.978 | 90.654 | 1920 |
-| gemini-3-flash-preview | 0.959 | 0.903 | 0.905 | 0.919 | 0.931 | 0.992 | 80.567 | 1920 |
-| gemini-3.5-flash | 0.953 | 0.896 | 0.901 | 0.912 | 0.922 | 0.986 | 92.296 | 1920 |
-| deepseek-v4-pro | 0.949 | 0.896 | 0.900 | 0.920 | 0.931 | 0.980 | 85.960 | 1920 |
-| grok-build-0.1 | 0.944 | 0.895 | 0.901 | 0.912 | 0.919 | 0.986 | 106.525 | 1920 |
-| claude-sonnet-4.6 | 0.945 | 0.897 | 0.903 | 0.924 | 0.931 | 0.949 | 63.579 | 1920 |
-| gemini-3.5-flash-lite | 0.930 | 0.898 | 0.901 | 0.907 | 0.912 | 0.960 | 174.048 | 1920 |
-| claude-opus-4.7 | 0.930 | 0.887 | 0.891 | 0.901 | 0.911 | 0.935 | 104.134 | 1920 |
-| glm-5.3 | 0.916 | 0.874 | 0.884 | 0.896 | 0.900 | 0.933 | 93.046 | 1920 |
-| grok-4.3 | 0.937 | 0.903 | 0.911 | 0.915 | 0.922 | 0.971 | 176.677 | 1920 |
-| gemini-3.1-flash-lite-preview | 0.932 | 0.892 | 0.900 | 0.906 | 0.912 | 0.976 | 161.521 | 1920 |
-| qwen-3.7-max | 0.915 | 0.872 | 0.878 | 0.888 | 0.892 | 0.951 | 171.021 | 1920 |
-| glm-5.2 | 0.885 | 0.844 | 0.848 | 0.866 | 0.873 | 0.913 | 82.945 | 1920 |
-| claude-opus-4.8 | 0.917 | 0.878 | 0.882 | 0.889 | 0.896 | 0.914 | 128.332 | 1920 |
-| claude-haiku-4.5 | 0.898 | 0.864 | 0.870 | 0.876 | 0.881 | 0.923 | 232.660 | 1920 |
-| qwen3.8-max | 0.898 | 0.864 | 0.867 | 0.873 | 0.878 | 0.935 | 255.392 | 1920 |
-| minimax-m3 | 0.912 | 0.883 | 0.890 | 0.902 | 0.902 | 0.962 | 297.407 | 1920 |
-| claude-sonnet-5 | 0.917 | 0.874 | 0.882 | 0.887 | 0.893 | 0.941 | 160.295 | 1920 |
-| gpt-5.4-mini | 0.890 | 0.858 | 0.867 | 0.869 | 0.874 | 0.918 | 242.369 | 1920 |
-| gpt-5.4-nano | 0.884 | 0.845 | 0.849 | 0.859 | 0.866 | 0.932 | 336.759 | 1920 |
-| kimi-k2.6 | 0.773 | 0.762 | 0.762 | 0.767 | 0.771 | 0.905 | 223.800 | 1920 |
+| gpt-6-sol | 0.983 | 0.962 | 0.966 | 0.972 | 0.976 | 0.996 | 19.200 | 1926 |
+| claude-opus-5.5 | 0.983 | 0.962 | 0.969 | 0.973 | 0.978 | 0.988 | 18.488 | 1926 |
+| gpt-6-luna | 0.977 | 0.947 | 0.955 | 0.968 | 0.972 | 0.989 | 25.183 | 1926 |
+| gpt-5.6-sol | 0.978 | 0.954 | 0.959 | 0.968 | 0.970 | 0.991 | 26.191 | 1926 |
+| gpt-6-astra | 0.981 | 0.953 | 0.958 | 0.964 | 0.967 | 0.988 | 30.755 | 1926 |
+| claude-fable-5.1 | 0.979 | 0.950 | 0.958 | 0.966 | 0.971 | 0.985 | 31.171 | 1926 |
+| claude-sonnet-5.5 | 0.970 | 0.943 | 0.950 | 0.959 | 0.962 | 0.984 | 31.411 | 1926 |
+| grok-4.7 | 0.975 | 0.928 | 0.942 | 0.955 | 0.963 | 0.984 | 39.734 | 1926 |
+| gemini-3.8-flash | 0.975 | 0.923 | 0.927 | 0.952 | 0.960 | 0.990 | 51.480 | 1926 |
+| inkling | 0.969 | 0.924 | 0.936 | 0.946 | 0.954 | 0.993 | 52.371 | 1926 |
+| gpt-6.1-sol | 0.979 | 0.951 | 0.953 | 0.961 | 0.968 | 0.985 | 29.052 | 1926 |
+| gemini-3.6-flash | 0.969 | 0.909 | 0.914 | 0.936 | 0.942 | 0.991 | 58.850 | 1926 |
+| gpt-5.5 | 0.966 | 0.926 | 0.932 | 0.946 | 0.955 | 0.975 | 47.418 | 1926 |
+| gpt-5.6-terra | 0.965 | 0.927 | 0.935 | 0.946 | 0.955 | 0.985 | 54.833 | 1926 |
+| ox-alpha | 0.962 | 0.925 | 0.934 | 0.944 | 0.954 | 0.979 | 51.147 | 1926 |
+| deepseek-v4.1-flash | 0.967 | 0.921 | 0.932 | 0.946 | 0.954 | 0.982 | 48.564 | 1926 |
+| kimi-k3 | 0.946 | 0.918 | 0.927 | 0.936 | 0.938 | 0.950 | 18.241 | 1926 |
+| grok-4.6 | 0.969 | 0.921 | 0.929 | 0.943 | 0.949 | 0.986 | 66.153 | 1926 |
+| gpt-5.6-luna | 0.959 | 0.927 | 0.935 | 0.943 | 0.949 | 0.982 | 88.970 | 1926 |
+| grok-4.5 | 0.962 | 0.916 | 0.921 | 0.933 | 0.943 | 0.988 | 71.953 | 1926 |
+| gemini-3.7-flash | 0.962 | 0.909 | 0.913 | 0.930 | 0.938 | 0.989 | 87.361 | 1926 |
+| deepseek-v4-pro-0813 | 0.963 | 0.912 | 0.922 | 0.934 | 0.941 | 0.993 | 91.626 | 1926 |
+| deepseek-v4-flash-0731 | 0.950 | 0.901 | 0.908 | 0.927 | 0.932 | 0.988 | 109.854 | 1926 |
+| gemini-3.1-pro-preview | 0.957 | 0.901 | 0.906 | 0.918 | 0.926 | 0.985 | 88.609 | 1926 |
+| claude-opus-5 | 0.954 | 0.909 | 0.913 | 0.923 | 0.935 | 0.982 | 113.432 | 1926 |
+| gemini-3-flash-preview | 0.961 | 0.900 | 0.903 | 0.919 | 0.931 | 0.992 | 91.593 | 1926 |
+| claude-fable-5 | 0.954 | 0.908 | 0.913 | 0.925 | 0.932 | 0.978 | 108.453 | 1926 |
+| gemini-3.5-flash | 0.955 | 0.894 | 0.900 | 0.912 | 0.924 | 0.986 | 94.695 | 1926 |
+| deepseek-v4-pro | 0.949 | 0.893 | 0.898 | 0.918 | 0.930 | 0.980 | 103.888 | 1926 |
+| grok-build-0.1 | 0.945 | 0.892 | 0.897 | 0.912 | 0.921 | 0.986 | 110.562 | 1926 |
+| claude-sonnet-4.6 | 0.944 | 0.895 | 0.900 | 0.923 | 0.930 | 0.949 | 75.755 | 1926 |
+| glm-5.3 | 0.916 | 0.871 | 0.882 | 0.893 | 0.898 | 0.933 | 104.805 | 1926 |
+| claude-opus-4.7 | 0.929 | 0.884 | 0.889 | 0.896 | 0.909 | 0.935 | 123.418 | 1926 |
+| gemini-3.5-flash-lite | 0.929 | 0.895 | 0.899 | 0.904 | 0.910 | 0.960 | 210.699 | 1926 |
+| gemini-3.1-flash-lite-preview | 0.933 | 0.890 | 0.898 | 0.904 | 0.911 | 0.976 | 189.913 | 1926 |
+| grok-4.3 | 0.936 | 0.900 | 0.908 | 0.913 | 0.921 | 0.971 | 205.357 | 1926 |
+| qwen-3.7-max | 0.914 | 0.870 | 0.876 | 0.885 | 0.891 | 0.951 | 226.424 | 1926 |
+| glm-5.2 | 0.883 | 0.841 | 0.846 | 0.864 | 0.871 | 0.913 | 115.306 | 1926 |
+| claude-haiku-5.5 | 0.921 | 0.892 | 0.898 | 0.903 | 0.908 | 0.964 | 356.148 | 1926 |
+| claude-opus-4.8 | 0.917 | 0.875 | 0.879 | 0.886 | 0.895 | 0.914 | 155.182 | 1926 |
+| claude-haiku-4.5 | 0.897 | 0.861 | 0.869 | 0.874 | 0.880 | 0.923 | 285.962 | 1926 |
+| qwen3.8-max | 0.898 | 0.862 | 0.865 | 0.872 | 0.878 | 0.935 | 285.159 | 1926 |
+| minimax-m3 | 0.910 | 0.881 | 0.887 | 0.899 | 0.901 | 0.962 | 370.752 | 1926 |
+| claude-sonnet-5 | 0.915 | 0.871 | 0.878 | 0.884 | 0.891 | 0.941 | 202.594 | 1926 |
+| gpt-5.4-mini | 0.888 | 0.856 | 0.865 | 0.867 | 0.871 | 0.918 | 306.592 | 1926 |
+| gpt-5.4-nano | 0.883 | 0.843 | 0.847 | 0.855 | 0.863 | 0.932 | 368.393 | 1926 |
+| kimi-k2.6 | 0.770 | 0.760 | 0.760 | 0.764 | 0.768 | 0.905 | 226.779 | 1926 |
## Population household-impact weights (headline)
@@ -120,52 +122,53 @@ Households receive equal weight. The score is a weighted average of row scores:
| model | bounded_score | amount_accuracy | participation_accuracy |
| --- | ---: | ---: | ---: |
-| claude-fable-5 | 0.930 | 0.808 | 0.967 |
-| claude-fable-5.1 | 0.967 | 0.895 | 0.983 |
-| claude-haiku-4.5 | 0.837 | 0.624 | 0.918 |
-| claude-opus-4.7 | 0.901 | 0.813 | 0.941 |
-| claude-opus-4.8 | 0.850 | 0.761 | 0.928 |
-| claude-opus-5 | 0.932 | 0.812 | 0.968 |
-| claude-opus-5.5 | 0.977 | 0.886 | 0.987 |
-| claude-sonnet-4.6 | 0.910 | 0.847 | 0.951 |
-| claude-sonnet-5 | 0.826 | 0.752 | 0.927 |
-| claude-sonnet-5.5 | 0.971 | 0.883 | 0.979 |
-| deepseek-v4-flash-0731 | 0.932 | 0.820 | 0.965 |
-| deepseek-v4-pro | 0.917 | 0.788 | 0.966 |
-| deepseek-v4-pro-0813 | 0.936 | 0.820 | 0.977 |
-| deepseek-v4.1-flash | 0.951 | 0.856 | 0.974 |
-| gemini-3-flash-preview | 0.928 | 0.800 | 0.977 |
-| gemini-3.1-flash-lite-preview | 0.892 | 0.721 | 0.952 |
-| gemini-3.1-pro-preview | 0.931 | 0.764 | 0.974 |
-| gemini-3.5-flash | 0.921 | 0.765 | 0.972 |
-| gemini-3.5-flash-lite | 0.901 | 0.644 | 0.945 |
-| gemini-3.6-flash | 0.954 | 0.847 | 0.981 |
-| gemini-3.7-flash | 0.936 | 0.781 | 0.976 |
-| gemini-3.8-flash | 0.964 | 0.842 | 0.982 |
-| glm-5.2 | 0.870 | 0.726 | 0.901 |
-| glm-5.3 | 0.898 | 0.796 | 0.931 |
-| gpt-5.4-mini | 0.817 | 0.498 | 0.909 |
-| gpt-5.4-nano | 0.765 | 0.497 | 0.902 |
-| gpt-5.5 | 0.952 | 0.880 | 0.970 |
-| gpt-5.6-luna | 0.946 | 0.832 | 0.968 |
-| gpt-5.6-sol | 0.970 | 0.887 | 0.983 |
-| gpt-5.6-terra | 0.953 | 0.863 | 0.973 |
-| gpt-6-astra | 0.971 | 0.908 | 0.986 |
-| gpt-6-luna | 0.975 | 0.888 | 0.982 |
-| gpt-6-sol | 0.981 | 0.903 | 0.986 |
-| gpt-6.1-sol | 0.957 | 0.897 | 0.982 |
-| grok-4.3 | 0.893 | 0.679 | 0.949 |
-| grok-4.5 | 0.938 | 0.809 | 0.976 |
-| grok-4.6 | 0.950 | 0.833 | 0.979 |
-| grok-4.7 | 0.969 | 0.881 | 0.981 |
-| grok-build-0.1 | 0.913 | 0.789 | 0.963 |
-| inkling | 0.965 | 0.855 | 0.978 |
-| kimi-k2.6 | 0.702 | 0.276 | 0.783 |
-| kimi-k3 | 0.946 | 0.853 | 0.951 |
-| minimax-m3 | 0.829 | 0.506 | 0.926 |
-| ox-alpha | 0.950 | 0.831 | 0.968 |
-| qwen-3.7-max | 0.875 | 0.726 | 0.937 |
-| qwen3.8-max | 0.832 | 0.597 | 0.916 |
+| claude-fable-5 | 0.926 | 0.799 | 0.968 |
+| claude-fable-5.1 | 0.969 | 0.905 | 0.982 |
+| claude-haiku-4.5 | 0.832 | 0.610 | 0.918 |
+| claude-haiku-5.5 | 0.862 | 0.525 | 0.936 |
+| claude-opus-4.7 | 0.897 | 0.808 | 0.940 |
+| claude-opus-4.8 | 0.846 | 0.757 | 0.929 |
+| claude-opus-5 | 0.928 | 0.807 | 0.967 |
+| claude-opus-5.5 | 0.980 | 0.896 | 0.988 |
+| claude-sonnet-4.6 | 0.907 | 0.840 | 0.952 |
+| claude-sonnet-5 | 0.821 | 0.741 | 0.927 |
+| claude-sonnet-5.5 | 0.969 | 0.880 | 0.980 |
+| deepseek-v4-flash-0731 | 0.931 | 0.823 | 0.966 |
+| deepseek-v4-pro | 0.914 | 0.787 | 0.967 |
+| deepseek-v4-pro-0813 | 0.935 | 0.818 | 0.978 |
+| deepseek-v4.1-flash | 0.952 | 0.858 | 0.975 |
+| gemini-3-flash-preview | 0.927 | 0.803 | 0.978 |
+| gemini-3.1-flash-lite-preview | 0.889 | 0.715 | 0.953 |
+| gemini-3.1-pro-preview | 0.931 | 0.771 | 0.975 |
+| gemini-3.5-flash | 0.922 | 0.774 | 0.972 |
+| gemini-3.5-flash-lite | 0.896 | 0.633 | 0.945 |
+| gemini-3.6-flash | 0.954 | 0.853 | 0.982 |
+| gemini-3.7-flash | 0.936 | 0.790 | 0.977 |
+| gemini-3.8-flash | 0.968 | 0.859 | 0.983 |
+| glm-5.2 | 0.864 | 0.728 | 0.899 |
+| glm-5.3 | 0.898 | 0.798 | 0.929 |
+| gpt-5.4-mini | 0.809 | 0.478 | 0.908 |
+| gpt-5.4-nano | 0.760 | 0.480 | 0.901 |
+| gpt-5.5 | 0.953 | 0.885 | 0.970 |
+| gpt-5.6-luna | 0.946 | 0.836 | 0.969 |
+| gpt-5.6-sol | 0.972 | 0.896 | 0.983 |
+| gpt-5.6-terra | 0.953 | 0.867 | 0.974 |
+| gpt-6-astra | 0.970 | 0.907 | 0.986 |
+| gpt-6-luna | 0.975 | 0.891 | 0.983 |
+| gpt-6-sol | 0.982 | 0.912 | 0.987 |
+| gpt-6.1-sol | 0.957 | 0.898 | 0.983 |
+| grok-4.3 | 0.889 | 0.670 | 0.949 |
+| grok-4.5 | 0.938 | 0.813 | 0.976 |
+| grok-4.6 | 0.950 | 0.835 | 0.979 |
+| grok-4.7 | 0.969 | 0.882 | 0.981 |
+| grok-build-0.1 | 0.913 | 0.795 | 0.963 |
+| inkling | 0.966 | 0.864 | 0.979 |
+| kimi-k2.6 | 0.694 | 0.276 | 0.780 |
+| kimi-k3 | 0.951 | 0.868 | 0.951 |
+| minimax-m3 | 0.821 | 0.497 | 0.924 |
+| ox-alpha | 0.952 | 0.844 | 0.969 |
+| qwen-3.7-max | 0.868 | 0.707 | 0.938 |
+| qwen3.8-max | 0.826 | 0.595 | 0.916 |
## Summary by variable
@@ -174,21 +177,21 @@ Amount variables use the tolerance columns. Binary coverage flags use `mean_accu
| variable | metric_type | mean_score | mean_exact | mean_within_1pct | mean_within_5pct | mean_within_10pct | mean_accuracy | mean_mae | total_n |
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
-| federal_income_tax_before_refundable_credits | amount_tolerance | 0.829 | 0.640 | 0.657 | 0.706 | 0.731 | n/a | 746.062 | 3634 |
-| federal_refundable_credits | amount_tolerance | 0.919 | 0.868 | 0.880 | 0.893 | 0.904 | n/a | 173.006 | 4508 |
-| free_school_meals_eligible | binary_accuracy | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.017 | 4600 |
-| local_income_tax | amount_tolerance | 0.993 | 0.993 | 0.993 | 0.993 | 0.993 | n/a | 6.366 | 4600 |
-| payroll_tax | amount_tolerance | 0.951 | 0.810 | 0.842 | 0.887 | 0.908 | n/a | 261.854 | 4370 |
-| person_chip_eligible | binary_accuracy | 0.958 | 0.958 | 0.958 | 0.958 | 0.958 | 0.958 | 0.036 | 8142 |
-| person_early_head_start_eligible | binary_accuracy | 0.983 | 0.983 | 0.983 | 0.983 | 0.983 | 0.983 | 0.011 | 1748 |
-| person_head_start_eligible | binary_accuracy | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.023 | 1748 |
-| person_medicaid_eligible | binary_accuracy | 0.922 | 0.922 | 0.922 | 0.922 | 0.922 | 0.922 | 0.072 | 7958 |
-| person_medicare_eligible | binary_accuracy | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.023 | 7912 |
-| person_wic_eligible | binary_accuracy | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.015 | 8142 |
-| reduced_price_school_meals_eligible | binary_accuracy | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.011 | 4600 |
-| self_employment_tax | amount_tolerance | 0.976 | 0.970 | 0.975 | 0.976 | 0.976 | n/a | 23.083 | 4600 |
-| snap | amount_tolerance | 0.895 | 0.842 | 0.855 | 0.874 | 0.880 | n/a | 221.958 | 4278 |
-| ssi | amount_tolerance | 0.987 | 0.977 | 0.981 | 0.984 | 0.985 | n/a | 77.869 | 4462 |
-| state_income_tax_before_refundable_credits | amount_tolerance | 0.844 | 0.651 | 0.674 | 0.728 | 0.765 | n/a | 204.398 | 3910 |
-| state_refundable_credits | amount_tolerance | 0.855 | 0.819 | 0.820 | 0.825 | 0.828 | n/a | 61.134 | 4508 |
-| tanf | amount_tolerance | 0.988 | 0.985 | 0.985 | 0.985 | 0.985 | n/a | 43.135 | 4600 |
+| federal_income_tax_before_refundable_credits | amount_tolerance | 0.826 | 0.606 | 0.629 | 0.684 | 0.714 | n/a | 950.725 | 3948 |
+| federal_refundable_credits | amount_tolerance | 0.918 | 0.869 | 0.881 | 0.893 | 0.903 | n/a | 182.408 | 4606 |
+| free_school_meals_eligible | binary_accuracy | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.016 | 4700 |
+| local_income_tax | amount_tolerance | 0.993 | 0.993 | 0.993 | 0.993 | 0.993 | n/a | 8.096 | 4606 |
+| payroll_tax | amount_tolerance | 0.948 | 0.799 | 0.832 | 0.879 | 0.905 | n/a | 293.480 | 4512 |
+| person_chip_eligible | binary_accuracy | 0.959 | 0.959 | 0.959 | 0.959 | 0.959 | 0.959 | 0.035 | 8319 |
+| person_early_head_start_eligible | binary_accuracy | 0.984 | 0.984 | 0.984 | 0.984 | 0.984 | 0.984 | 0.011 | 1786 |
+| person_head_start_eligible | binary_accuracy | 0.970 | 0.970 | 0.970 | 0.970 | 0.970 | 0.970 | 0.025 | 1786 |
+| person_medicaid_eligible | binary_accuracy | 0.921 | 0.921 | 0.921 | 0.921 | 0.921 | 0.921 | 0.073 | 8131 |
+| person_medicare_eligible | binary_accuracy | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.023 | 8084 |
+| person_wic_eligible | binary_accuracy | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.016 | 8319 |
+| reduced_price_school_meals_eligible | binary_accuracy | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.011 | 4700 |
+| self_employment_tax | amount_tolerance | 0.977 | 0.970 | 0.976 | 0.976 | 0.977 | n/a | 22.595 | 4700 |
+| snap | amount_tolerance | 0.894 | 0.842 | 0.855 | 0.874 | 0.880 | n/a | 228.335 | 4371 |
+| ssi | amount_tolerance | 0.987 | 0.977 | 0.981 | 0.984 | 0.984 | n/a | 80.585 | 4559 |
+| state_income_tax_before_refundable_credits | amount_tolerance | 0.844 | 0.640 | 0.663 | 0.719 | 0.765 | n/a | 295.257 | 4089 |
+| state_refundable_credits | amount_tolerance | 0.862 | 0.828 | 0.828 | 0.833 | 0.837 | n/a | 66.475 | 4606 |
+| tanf | amount_tolerance | 0.988 | 0.985 | 0.985 | 0.985 | 0.986 | n/a | 43.507 | 4700 |
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv
index 2662d573..4c504517 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv
@@ -1,47 +1,48 @@
model,mean_score,mean_exact,mean_within_1pct,mean_within_5pct,mean_threshold_score,mean_mae,mean_mape,mean_within_10pct,mean_accuracy,mean_coverage,total_n,parsed_n,bounded_score,amount_accuracy,participation_accuracy,weighted_exact,weighted_within_1pct,weighted_within_5pct,weighted_within_10pct,weighted_threshold_score,equal_score,aggregate_score
-gpt-6-sol,0.9815523732534513,0.9618804869802869,0.9657584112728205,0.9726011483676584,0.9685495061600291,19.56679142156725,0.18499510258085217,0.9739579780193502,0.9964036119003299,1.0,1920,1920,0.9808184340618006,0.9033910702690353,0.9864583333333333,0.958278593391396,0.9648909081938797,0.970509323505921,0.9732703639449344,0.9667372972590327,0.9813086035509729,0.9773084479983554
-claude-opus-5.5,0.981643884689707,0.9627060080142082,0.9689931514178913,0.9729272989165533,0.9701665575811513,24.878105401567712,0.12064391759845261,0.9760397719759525,0.9884163237647366,1.0,1920,1920,0.9771690147254536,0.8856148704783553,0.9869791666666666,0.950517690085572,0.9602921441673774,0.9662545183935254,0.9708278828166713,0.9619730588657867,0.982665717363308,0.9672794672942899
-gpt-6-luna,0.9771517258559326,0.9490272665506131,0.9571740883519806,0.9696796138453416,0.9622319591669756,21.62457938431898,0.1436195879621112,0.9730468679199673,0.9890609499636592,1.0,1920,1920,0.9746532416234666,0.8884189807189904,0.9822916666666667,0.9302067960711825,0.9452338792517015,0.9644938113747166,0.970745614767428,0.9526700253662572,0.9762963614412764,0.9670470194873013
-claude-sonnet-5.5,0.9705614621465083,0.94575436788247,0.9513238692091325,0.9606356952053537,0.9552486218390224,26.21334391926426,0.34411220436399703,0.9632805550591333,0.9843879780645893,1.0,1920,1920,0.9709879703434949,0.883027842153388,0.9791666666666666,0.9325308629240557,0.9406504558619261,0.9562633491349097,0.9632419719867577,0.9481716599769123,0.9712197435026624,0.9655519639970457
-gpt-6-astra,0.981242654062717,0.9550642009089627,0.961208417131577,0.9661249162650969,0.9627670075515241,23.407819778206537,0.3292577953683226,0.9686704959004597,0.9883762938567011,1.0,1920,1920,0.9706155505776884,0.9079517092466115,0.9864583333333333,0.9253539056444686,0.931249103766525,0.9430916948615062,0.947306121009987,0.9367502063206217,0.9792321350326181,0.9666765854610052
-gpt-5.6-sol,0.976897137832231,0.9569066225064792,0.9605919821364706,0.9693625078906319,0.9643827024916742,29.82167143720124,0.16048487389257157,0.9706696974331156,0.9906143887421841,1.0,1920,1920,0.9696398655249554,0.8871449705485469,0.9828125,0.9443315328635904,0.9490984614586152,0.9622040012859511,0.9634584171198673,0.9547731031820059,0.9764359621352482,0.9648161177256759
-grok-4.7,0.9737946513705028,0.9306074650556799,0.9434987926220256,0.9564802945410733,0.9482923219340608,39.821288956164345,0.1707074422899231,0.962582735517464,0.9843209355108935,1.0,1920,1920,0.9686156084070919,0.8813809791557161,0.9807291666666667,0.8929738601164692,0.9182497624877035,0.9459185081993605,0.9509834611302109,0.9270313979834361,0.9753563722101339,0.9513006736499069
-claude-fable-5.1,0.9783552474129373,0.9521009691470409,0.9596808948966371,0.9677722326975804,0.9627514239805686,37.64244231592298,0.22634388925184146,0.9714515991810158,0.9847457446743875,1.0,1920,1920,0.9665274167005613,0.8946044359971738,0.9828125,0.92070046378755,0.9321454952972035,0.9468925742151069,0.9516005405523782,0.9378347684630596,0.9772268847155985,0.9612071789126171
-inkling,0.9679868490905835,0.9273963504063043,0.9378782852828914,0.9465542665223973,0.9412934377085074,50.09414824841062,0.23654944908551587,0.9533448486224368,0.9931973972110643,1.0,1920,1920,0.9647956124455398,0.8550178740482619,0.978125,0.8932869677000899,0.9095535550808188,0.9235887594799701,0.9372144712141902,0.9159109383687672,0.9675229518781202,0.9504148657331077
-gemini-3.8-flash,0.971930104758894,0.9253388871071171,0.9296209508097911,0.9510425973221458,0.9407505458385016,56.119611304681186,0.30417653308432696,0.9569997481149524,0.989812424479932,1.0,1920,1920,0.9642171091422845,0.8419676821147123,0.9817708333333334,0.8721883965470891,0.8801480893014066,0.9229998193188487,0.935058307175155,0.9025986530856248,0.9730875336850235,0.9441158524303325
-gpt-6.1-sol,0.9788636406702812,0.9536207199793792,0.9555252810869762,0.9616993296537716,0.959545888432066,24.45907347553863,0.11653621286855752,0.9673382230081374,0.9854903212797419,1.0,1920,1920,0.9565738864426535,0.8968186061739869,0.9822916666666667,0.912629648595955,0.9142986993457913,0.925969117995431,0.9363626131194239,0.9223150197641503,0.9764180515850194,0.9539219209293924
-gemini-3.6-flash,0.968213332192506,0.9113381801563336,0.9162935192063588,0.9361784189903415,0.9265902865867509,59.64388230257852,0.26278067416521655,0.9425510279939695,0.9908836664380654,1.0,1920,1920,0.9536998582624355,0.8466742517913447,0.98125,0.8420619812334691,0.8488689403867585,0.8879875842366797,0.9049335711910799,0.8709630192619967,0.9690303576115015,0.9299627583321886
-gpt-5.6-terra,0.9639928498455304,0.9293923454207705,0.9375641708246376,0.9476166649196797,0.9423135921351073,53.157946714349464,0.33975813650141007,0.9546811873753419,0.9849007217269194,1.0,1920,1920,0.9526259093174834,0.8634935732549271,0.9729166666666667,0.88968823575796,0.9030421864199438,0.9222406529757445,0.9345509232647004,0.9123804996045871,0.9629954021656948,0.9454362577712131
-gpt-5.5,0.9648413279914065,0.9275164073528285,0.9323658830875173,0.9462209876432193,0.9400047895074243,47.77776958494626,0.20490260187177234,0.9539158799461318,0.9748611332623985,1.0,1920,1920,0.9524491710488685,0.8804686635883783,0.9697916666666667,0.8910223705999104,0.8964063272344279,0.9228563375479599,0.9345900177910452,0.9112187632933358,0.9659926700982111,0.9402512399515769
-deepseek-v4.1-flash,0.9660228377722352,0.9239238654792162,0.9323336781881637,0.9470038740862081,0.9393496534362656,44.58741206592821,0.21079248865999134,0.9541371959914748,0.9818471695970921,1.0,1920,1920,0.9513044683042773,0.8558185110238472,0.9744791666666667,0.8800840767222912,0.8901863655593347,0.9194765097079017,0.9301807704406788,0.9049819306075516,0.9665104039971056,0.9316084360575165
-grok-4.6,0.9675918345901326,0.9239069761418224,0.930597539188629,0.9437110162129678,0.9367702958625816,62.46918919732448,0.22451862183674962,0.9488656519069077,0.9861025929917377,1.0,1920,1920,0.9502068169526006,0.8327498632341238,0.9786458333333333,0.8751926844685189,0.8862719861967554,0.9097112738098244,0.9170223456435859,0.8970495725296712,0.9695494149940004,0.9203708341003033
-ox-alpha,0.9603019695918482,0.9268680550860782,0.9353587248801277,0.9444019335471167,0.9396269548965308,53.54311519725835,0.3044458082746513,0.9518791060728002,0.9793121180326922,1.0,1920,1920,0.9499767053343539,0.8311461799407752,0.9682291666666667,0.8977515163322984,0.9080486931574937,0.9252766866844236,0.9344586409553155,0.9163838842823827,0.9614888423489996,0.9335414722616632
-kimi-k3,0.9454326495159084,0.9192392730826564,0.9273681903586879,0.9363067957808702,0.9299469871067801,26.155894278536373,0.1858655192857958,0.9368736892049063,0.9496126351396752,0.9690830663672815,1920,1862,0.9464881582201117,0.8525302827265958,0.9510416666666667,0.9122815783106216,0.9227606963894561,0.9371751604396521,0.9372407182878396,0.9273645383568924,0.9461988490584059,0.9434473374448828
-gpt-5.6-luna,0.9584953228884283,0.9293538978456175,0.9362634190446484,0.9443558650330467,0.9400247390282535,88.47943601930172,0.29751223578181685,0.9501257741897012,0.9820753882994114,1.0,1920,1920,0.9462741567619949,0.8315888698779782,0.9682291666666667,0.9027941661573834,0.9150752683157661,0.9267573567793618,0.9363302232449726,0.920239253624371,0.9567685049001201,0.9334938341194896
-grok-4.5,0.9599538433262739,0.9190303234171814,0.9244454351331114,0.934290417065528,0.9297741575356224,64.33559083640272,0.27795682951966666,0.941330454526669,0.9883304796146132,1.0,1920,1920,0.9380870728162982,0.8092778524401018,0.9755208333333333,0.8629327728338254,0.8722067896864145,0.8917637130578866,0.9035917853787058,0.882623765239208,0.9596660728158796,0.9048497238715081
-deepseek-v4-pro-0813,0.9619081735149818,0.914541498053199,0.9224525492952836,0.9330191866137297,0.9276146738874057,80.24276834371092,0.37164059944412764,0.94044546158741,0.9929817810090811,1.0,1920,1920,0.9361519414356262,0.8202272949654394,0.9770833333333333,0.8464638044169422,0.8586345313771789,0.8781416077666516,0.8906659291720359,0.8684764681832021,0.9618471287497634,0.8949307160923826
-gemini-3.7-flash,0.9602213946911573,0.9118871272897593,0.915690777594111,0.9310204121546134,0.9238051403273665,81.96975579493962,0.37665737315169034,0.9366222442709822,0.9889274517487998,1.0,1920,1920,0.936070022347849,0.7807355865158802,0.9760416666666667,0.8421804200586138,0.8473712854368017,0.8771156285794365,0.8892381923351664,0.8639763816025047,0.9608564928752874,0.9037649694972688
-deepseek-v4-flash-0731,0.9492996896230168,0.9035769686426222,0.9108541056895744,0.929345415351395,0.9190662140588657,102.56849991797134,0.3994067295022672,0.9324883665518714,0.9876405935886833,1.0,1920,1920,0.9322053278544827,0.8196270176229895,0.9651041666666667,0.8451911095150071,0.8542651871837313,0.8943035168206734,0.898177837231463,0.8729844126877186,0.9486462889123586,0.9024453950436349
-claude-opus-5,0.9550162577443037,0.9111942523353499,0.9152520157916094,0.9262262007514125,0.9221738551947156,86.75963365570121,0.548287099357943,0.9360229519004906,0.9818555712639828,1.0,1920,1920,0.9319210055033583,0.8122723892348033,0.9677083333333333,0.8522255993736841,0.8620061062972073,0.8797369934264353,0.8956710799886478,0.8724099447714937,0.9529434341309628,0.914351897428697
-gemini-3.1-pro-preview,0.9552315582297108,0.9032806172215467,0.9069229224669849,0.9184288544080006,0.9136061420726781,83.41216435948277,0.3573801134181219,0.9257921741941801,0.9850067329254645,1.0,1920,1920,0.9311109940854441,0.7640380887946329,0.9744791666666667,0.8276615889413155,0.8334471109605552,0.8550675420877831,0.8682523512161903,0.846107148301461,0.9577177802422768,0.8942146382501285
-claude-fable-5,0.9546921300229511,0.9112071582143552,0.9153757792492672,0.9278362540804631,0.9223267004715926,90.65400614476727,0.4047695117080156,0.9348876103422852,0.9779773029680953,1.0,1920,1920,0.92972816559178,0.8078974928658637,0.9671875,0.8489232434256384,0.8531574213446198,0.8788916859938192,0.8937169365118148,0.868672321818973,0.9517636458354639,0.9135500946731699
-gemini-3-flash-preview,0.9593539200882888,0.9027305592655007,0.9052390776628663,0.9193469639575608,0.9145325549959514,80.56665682703499,0.25762487498367453,0.930813619097878,0.9915530229672144,1.0,1920,1920,0.9275432219949714,0.7995091660799631,0.9770833333333333,0.8223561003657501,0.8270360256915887,0.8574386839673035,0.8739417828142693,0.8451931482097279,0.9584292648644067,0.8925584001210612
-gemini-3.5-flash,0.9529829514405252,0.8964904393605099,0.9013895552044067,0.9118251767205643,0.9079299432842443,92.29622691511207,0.27042262966684283,0.9220146018514966,0.9862444928734878,1.0,1920,1920,0.9214069205791094,0.7646419129186436,0.971875,0.813347797137955,0.8194404941231922,0.8400067180645132,0.8602614261217459,0.8332641088618515,0.953261709815471,0.8796273320579268
-deepseek-v4-pro,0.9485157703504004,0.8956815394602639,0.8998635867374395,0.9195734570315776,0.9114912508708447,85.9602884570127,0.3094518769419913,0.9308464202540979,0.9802109464940681,1.0,1920,1920,0.9166562664125493,0.7878627283025389,0.9661458333333334,0.808234097199763,0.8181552420426673,0.8620749608318506,0.8848068937867187,0.8433177984652499,0.9494840121914642,0.8817138658612726
-grok-build-0.1,0.943727820271762,0.8954781848942073,0.9009040941751051,0.9115472278538879,0.9068333856626478,106.52512254711168,0.36473082158677644,0.9194040357273912,0.9856680502360821,1.0,1920,1920,0.9130139220843532,0.7886236475393988,0.9630208333333333,0.8161932318022014,0.8271239179707865,0.8477021364086919,0.8603057358179947,0.8378312554999187,0.9434004839674891,0.8650409330212756
-claude-sonnet-4.6,0.9445283306408988,0.8973456197973788,0.9029361140452669,0.9236469083629169,0.9136244167696965,63.5791190738884,0.3946999686182645,0.9305690248732237,0.9490938895443671,1.0,1920,1920,0.9096309186972799,0.8471209463194483,0.9510416666666667,0.8149661043992744,0.8256251320015275,0.877812093423913,0.8862596923443321,0.8511657555422617,0.9468455364296066,0.9121355261355109
-gemini-3.5-flash-lite,0.929988388664647,0.8976730565793642,0.9012417885358377,0.906954217043569,0.9045158397001727,174.04758144993332,0.6725741232062935,0.9121942966419196,0.9600548342296911,1.0,1920,1920,0.9011346811653275,0.6435573681078058,0.9447916666666667,0.8319892957661855,0.8390858702927438,0.8524675451496029,0.8649250103809186,0.8471169303973628,0.9353306083786298,0.8663760234198441
-claude-opus-4.7,0.9301410373850852,0.8870317473815054,0.891432250642992,0.900680453588394,0.8976564212097357,104.13404808852948,0.948534535148469,0.911481233226051,0.9345062340618155,1.0,1920,1920,0.9006441372692923,0.8134830804335053,0.940625,0.82163216319256,0.8290840381362165,0.8507045274179823,0.863834607123123,0.8413138339674705,0.9356621063083834,0.9035612993922874
-glm-5.3,0.9163237804261208,0.8742183009359465,0.8842821112039346,0.8958841484976632,0.8885318685218363,93.04606319753326,0.47635699426563627,0.8997429134498005,0.9333861713989211,0.9625036222464023,1920,1849,0.8980724721207594,0.7957508955759104,0.9307291666666667,0.823445616390977,0.8392869180840702,0.8621023303180121,0.8711470544183334,0.8489954798028482,0.9227062503455279,0.8754783318086635
-grok-4.3,0.9366101255315716,0.902549552165197,0.9110730854959177,0.9150251348921807,0.9125798642011819,176.67684473708752,0.6915618725482697,0.9216716842514315,0.9706701665167907,1.0,1920,1920,0.8925945161766188,0.6786019473503241,0.9489583333333333,0.8188053476681135,0.8379799192824424,0.8463456030599096,0.8624616217037421,0.841398122928552,0.9377097641476108,0.8674282964095444
-gemini-3.1-flash-lite-preview,0.932419727186675,0.8921543503672729,0.8999542484744172,0.9059179632963441,0.9024922665877297,161.52093387423875,0.6866935015184725,0.9119425042128851,0.9755600078477515,1.0,1920,1920,0.8919628225969336,0.721233762833077,0.9520833333333333,0.8094050297084618,0.8268549044301614,0.8399404084352,0.8522233671830022,0.8321059274392063,0.93391354413219,0.8413170393880475
-qwen-3.7-max,0.9149940378558166,0.8724382608703568,0.8778019564263052,0.8879333113288377,0.8826516558585601,171.02129735258325,0.596122388948601,0.8924330948087409,0.9510034528727395,1.0,1920,1920,0.8751871722223891,0.7257846058980444,0.9369791666666667,0.7860647395672627,0.8001471132751776,0.8236517860381483,0.83542277958954,0.8113216046175321,0.9219428262023415,0.8321141395880445
-glm-5.2,0.8849227327427087,0.8435305105683888,0.8477054535129313,0.866052060313797,0.8574733367490761,82.94500187476811,0.5937475281029702,0.8726053226011872,0.9129817267467543,0.9285558562084442,1920,1788,0.8700225780157449,0.7258888833394369,0.9005208333333333,0.780090780621868,0.7899650712075283,0.8319262431871857,0.8449719118403135,0.811738501714224,0.8923114780258545,0.8463083815877664
-claude-opus-4.8,0.9174198110215743,0.8776580053571487,0.8815220959634111,0.8885495571810761,0.8860225075077781,128.3316567521644,0.6053027943323444,0.8963603715294766,0.9144716041047022,1.0,1920,1920,0.8499381291144472,0.7613433484931525,0.928125,0.7737850060971023,0.7845917945527894,0.7975838655692756,0.8151397808957895,0.7927751117787394,0.9208260039822946,0.8577711798396982
-claude-haiku-4.5,0.8975616317126833,0.8639018012521774,0.8704340695003606,0.8756053496491067,0.8727960514905806,232.6604433222701,0.671356153997025,0.8812429855606777,0.92324343989455,1.0,1920,1920,0.83684620419693,0.6242341838988358,0.9177083333333333,0.7588064801985028,0.7721026656684341,0.7846907951139509,0.7991981368919405,0.778699519468207,0.9029988310725265,0.8111634763994128
-qwen3.8-max,0.8981342605023754,0.8640880609378851,0.8671496364452811,0.8734229652304927,0.8707570871981782,255.3918618258527,0.9750536369772678,0.8783676861790541,0.9346093562161663,1.0,1920,1920,0.832059811806064,0.5965432867381539,0.915625,0.7640969244962698,0.7701592609415729,0.7840800476563353,0.792491815410051,0.7777070121260573,0.9039413518606448,0.8002827905961581
-minimax-m3,0.9118206122595388,0.8829858983826371,0.8896658459557147,0.9015284115460194,0.894076484748876,297.4069766280429,0.842209831441688,0.9021257831111327,0.961559287018209,1.0,1920,1920,0.8287336744870811,0.5059437867647049,0.9255208333333333,0.7689602609069552,0.784208059812372,0.8118922697818669,0.8123069046977869,0.7943418737997453,0.9121063828473147,0.778029767410614
-claude-sonnet-5,0.916656336616104,0.8739031370177606,0.8816748133098629,0.8865734105457815,0.8836701655353658,160.2950691101056,1.0909774754897312,0.8925293012680583,0.9405477461607955,1.0,1920,1920,0.8256854338114493,0.7518096806066631,0.9270833333333334,0.7379393638195099,0.7581444232127601,0.7679562467893192,0.7806171164078397,0.7611642875573572,0.916010701833311,0.824341292729381
-gpt-5.4-mini,0.8903893039962552,0.8582791869809641,0.8674897132967535,0.8693816981608745,0.8672032757148364,242.36911896198797,0.7731288478339491,0.8736625044207537,0.9179161871995511,1.0,1920,1920,0.8174441029446535,0.49760059533068873,0.909375,0.7474882415461991,0.7671855843244875,0.77004808090877,0.7825780771826406,0.7668249959905242,0.9019786440328343,0.7825933344136378
-gpt-5.4-nano,0.8842850907152262,0.8450558740940551,0.849238641138248,0.8587329651836556,0.8547215871480975,336.7592376138673,0.8475378959081858,0.8658588681764317,0.931785064965096,1.0,1920,1920,0.7648719022083158,0.4967651706194354,0.9020833333333333,0.6653551716384947,0.6756687766954123,0.701094996924366,0.7182672979607428,0.6900965608047539,0.8877063409820727,0.7262350819027238
-kimi-k2.6,0.772609633180548,0.761729482328527,0.7623022200146667,0.7669509211944598,0.765432522675958,223.8000695540359,0.34727207760822065,0.7707474671661781,0.9047195412668892,0.7933913916590262,1920,1537,0.7015831669810351,0.27550560101579513,0.7828125,0.6811125273246809,0.6813280164754684,0.69242877037135,0.6970697272234028,0.6879847603487255,0.7583422014219023,0.6341126941346399
+gpt-6-sol,0.9830270517517401,0.962394547786723,0.9655201337027111,0.971641931080666,0.9689285357484879,19.200387502942093,0.176078458507419,0.9761575304238517,0.9964036119003299,1.0,1926,1926,0.9823310202092435,0.912082542779077,0.9870197300103842,0.9558331934063155,0.9615975832935518,0.9682309238137915,0.9757995796893004,0.9653653200507398,0.98262530596485,0.9802128956868253
+claude-opus-5.5,0.9834983677884005,0.9620905753498891,0.9689209232967553,0.9733648464598903,0.9706702866188979,18.488224132399388,0.10771036908547377,0.9783048013690571,0.9884163237647366,1.0,1926,1926,0.9799867306385405,0.8963402551927546,0.9875389408099688,0.9441227586954132,0.958024566045827,0.9649186525504183,0.973291048581783,0.9600892564683603,0.9842294198585403,0.9732199935650364
+gpt-6-luna,0.9774237105236696,0.9466426572543356,0.9552729238771137,0.9681427393351224,0.9605195223851981,25.182826375326954,0.14148152600528713,0.9720197690742212,0.9890609499636592,1.0,1926,1926,0.9747765365646219,0.8907871381564434,0.9828660436137072,0.9197473566861318,0.9387398059495254,0.9607197171680747,0.9677555301489587,0.9467406024881727,0.9765012194338702,0.967153799082312
+gpt-5.6-sol,0.9783472290156706,0.9542318406079683,0.9591830671744815,0.9677630317273525,0.9628738124133756,26.191252580768772,0.1513049136612662,0.9703173101437,0.9906143887421841,1.0,1926,1926,0.9718364579174521,0.8955394679451782,0.9833852544132918,0.9348850920910052,0.9436159306980213,0.9580857997288014,0.9605798149659428,0.9492916593709427,0.9775774305481384,0.9692172534088573
+gpt-6-astra,0.9806990533125602,0.9532186741026424,0.9580195588305292,0.9641169211471036,0.9606569658511795,30.75451966784898,0.07935524129081416,0.9672727093244434,0.9883762938567011,1.0,1926,1926,0.9696384987275308,0.9074828891402473,0.985981308411215,0.9194318916414569,0.9244100546973958,0.9370641989989815,0.9429653610536789,0.9309678765978782,0.9784461578204635,0.9643846957473317
+claude-fable-5.1,0.9792758576448009,0.9498587929251108,0.9579668980507354,0.9664051392475973,0.9613703605357169,31.171220509489917,0.20228604629167612,0.9712506119194246,0.9847457446743875,1.0,1926,1926,0.9690373062101615,0.9048302591722842,0.9823468328141225,0.9126770068917941,0.9283467520215154,0.9452407110111255,0.9518452577856976,0.9345274319275332,0.9778546425232635,0.9666762743881243
+claude-sonnet-5.5,0.9703944653893409,0.94337260051748,0.9495421907735552,0.9585245193371645,0.9534359834902464,31.410755759951567,0.34616884231272965,0.9623046233327857,0.9843879780645893,1.0,1926,1926,0.968880093375203,0.8795717997327471,0.9797507788161994,0.9215666336950988,0.9335077604027184,0.9494964080652072,0.9599025016948177,0.9411183259644605,0.9709707846164748,0.9610307544382658
+grok-4.7,0.9748886759556314,0.9283539299248971,0.9421112110477036,0.9546424038486404,0.9470885870710334,39.734466821391536,0.15690969625490322,0.9632468034628923,0.9843209355108935,1.0,1926,1926,0.9686981049536972,0.8824351850685971,0.9813084112149533,0.8807351092648591,0.9091406024792953,0.9385843881943866,0.9477231193859175,0.9190458048311146,0.9761947026454145,0.9508623402980265
+gemini-3.8-flash,0.974794453758634,0.9225434661527255,0.9274235551615091,0.9515473024568206,0.9403298638053855,51.480332170696094,0.2828324071055074,0.959805131450487,0.989812424479932,1.0,1926,1926,0.9679817809420698,0.8593774004252831,0.9828660436137072,0.8601161106925648,0.870569227220639,0.9183292114616621,0.9368075764240033,0.8964555314497172,0.9754713451200979,0.9520248184451073
+inkling,0.9693982590245758,0.9241717146906383,0.9357874728774873,0.9462353665016283,0.9400985303943639,52.370687846446486,0.22833449650145793,0.9541995675077014,0.9931973972110643,1.0,1926,1926,0.9658909534951285,0.8640502816666282,0.9787123572170301,0.8816475777106672,0.9021639905350343,0.9209935694666733,0.9364729608424082,0.9103195246386957,0.9685907878639913,0.9532233976895769
+gpt-6.1-sol,0.9793098796538511,0.9511381491842594,0.9529527940571635,0.9609658651859981,0.9582160208872603,29.052435903116773,0.11105816247756274,0.9678072751216198,0.9854903212797419,1.0,1926,1926,0.957026728677903,0.8979298163418677,0.9828660436137072,0.9034973390129966,0.9061040412868845,0.920607668883033,0.9346776378884012,0.9162216717678288,0.9766834752259455,0.9543162871661883
+gemini-3.6-flash,0.9694184004868391,0.9087478688391512,0.9136101815507032,0.9362625195233796,0.9252622174812635,58.84965127363211,0.2463140815422405,0.9424283000118202,0.9908836664380654,1.0,1926,1926,0.9537036823001062,0.8528757290451731,0.9818276220145379,0.8299602782682469,0.8367672374215364,0.8823236380488036,0.8985186428617218,0.8618924491500772,0.9696864129308805,0.9299574683967735
+gpt-5.5,0.9662282521395384,0.9255946874407756,0.9323315109256151,0.9458371766018472,0.9395862613840635,47.417906048720255,0.1910395487859728,0.9545816705680159,0.9748611332623985,1.0,1926,1926,0.9534563918168621,0.8845764248090755,0.9704049844236761,0.8823067085467435,0.8906359596748408,0.9184870710019918,0.931761029361829,0.9057976921463513,0.9668258425152148,0.9418155292053857
+gpt-5.6-terra,0.9650265814751435,0.9273705669823167,0.9353823620489169,0.9457677474489439,0.9407763521443466,54.83314239070233,0.3290132096341398,0.9545847320972093,0.9849007217269194,1.0,1926,1926,0.9526460975242655,0.867439208763258,0.9735202492211839,0.8801412866207078,0.8953249778258201,0.9149877804039745,0.9304844683102104,0.9052346282901783,0.963677944194106,0.9452353664551406
+ox-alpha,0.9623222100033453,0.9254865021196557,0.9338780539572985,0.9440067941506938,0.9393029068950128,51.14660633282609,0.2923061915290818,0.9538402773524035,0.9793121180326922,1.0,1926,1926,0.9519270805020915,0.8436519063360042,0.9688473520249221,0.890098106668138,0.9010676704844615,0.921998967294138,0.9343820748405102,0.911886704821812,0.9631078951903809,0.9378475719036602
+deepseek-v4.1-flash,0.9670067192778481,0.9209487910084039,0.9318744014590246,0.9461655886730029,0.9381499332540467,48.5641887514562,0.2007403724498377,0.9536109518757553,0.9818471695970921,1.0,1926,1926,0.9517752450059543,0.8583038385515398,0.9750778816199377,0.8676144526466474,0.8835849043058341,0.913332586795634,0.9251779194175973,0.8974274657914282,0.9671937047882954,0.932819995617995
+kimi-k3,0.9462586612251838,0.9180628993479297,0.9267032815606344,0.936027293732553,0.9296719017657922,18.24116924121017,0.17520097259250117,0.9378941324220518,0.9496126351396752,0.9678242667364074,1926,1866,0.9505238459925692,0.8682796383114129,0.95067497403946,0.9096808234088384,0.9220142985722447,0.9389211255328521,0.9416036292731564,0.928054969196773,0.9481198463458672,0.9516559391713265
+grok-4.6,0.9692112590658931,0.9211619402003678,0.9290451753149931,0.9431318025206039,0.9355300266869568,66.15325996751419,0.20680724088668695,0.948781188711862,0.9861025929917377,1.0,1926,1926,0.9501567270433771,0.8347805035536361,0.9792315680166147,0.8624587169800553,0.874773714192772,0.9012271182335156,0.9091653979842352,0.8869062368476446,0.9705050010780986,0.9197156365618397
+gpt-5.6-luna,0.95940181939668,0.9265690914006852,0.934633552695067,0.943183314986586,0.938297318125219,88.97015855872063,0.2872251225614467,0.9488033134185382,0.9820753882994114,1.0,1926,1926,0.9458495377892646,0.836435219714905,0.9688473520249221,0.8916352787289538,0.9080331511036889,0.922022773673525,0.9315205114554975,0.9133029287404163,0.95747521267871,0.9326979289902653
+grok-4.5,0.9618263304747465,0.9163543550978237,0.9210557379696362,0.9332957462891177,0.9283417679340782,71.95282395626856,0.26267080343015725,0.9426612323797349,0.9883304796146132,1.0,1926,1926,0.9381504426131009,0.8132376366454842,0.9761163032191069,0.8505194079916353,0.8596231685304276,0.8809971444531318,0.8977464489776125,0.8722215424882017,0.960905760658788,0.9048583361862801
+gemini-3.7-flash,0.9619514045989397,0.9092574577379425,0.9129741733217548,0.929906186363036,0.9225284207017008,87.36087209355702,0.3560659857962258,0.9379758653840697,0.9889274517487998,1.0,1926,1926,0.9363027660186353,0.7895223135672234,0.9766355140186916,0.8310818106563254,0.8362726760345132,0.8693401665211989,0.8860483010845296,0.8556857385741418,0.9617248826254058,0.904342716087635
+deepseek-v4-pro-0813,0.9630879103873716,0.9120995666520513,0.9224654040029735,0.9340999071133648,0.9275139499866959,91.62625949530089,0.35798245339587453,0.9413909221783945,0.9929817810090811,1.0,1926,1926,0.9345791264488473,0.8175366302746644,0.9776739356178609,0.8339367778864432,0.84906661136398,0.8728518061541278,0.8846094027080046,0.8601161495281389,0.9622959210544447,0.8916702520794096
+deepseek-v4-flash-0731,0.9502710914051025,0.9010609676504971,0.9076004171369557,0.9274674226154542,0.9171416657671513,109.85439639022488,0.3831292910288196,0.932437855665698,0.9876405935886833,1.0,1926,1926,0.9314196111818078,0.8233236396253075,0.9657320872274143,0.8330394017677412,0.8420119411936929,0.8846152063469203,0.8926685675596029,0.8630837792169893,0.9490952317003672,0.9005244335196241
+gemini-3.1-pro-preview,0.9569120891494663,0.9008101996990123,0.9056666681484611,0.9176502510712248,0.91241323325519,88.6094285271116,0.3432627195605471,0.9255258141020616,0.9850067329254645,1.0,1926,1926,0.9307013087371321,0.7709032849419969,0.9750778816199377,0.8157663599057944,0.8228714153530556,0.8465870437811653,0.8598164556906326,0.8362603186826619,0.9584514592819766,0.8933676147718944
+claude-opus-5,0.9543952904346727,0.9085180506426583,0.9131839256133362,0.9231989770856617,0.9198689708618151,113.43231436366587,0.4831671311163763,0.934574930105604,0.9818555712639828,1.0,1926,1926,0.9280544324678114,0.8071841189718817,0.9672897196261683,0.8403196452714528,0.8512530297861198,0.8686466236734707,0.8885695790092588,0.8621972194350755,0.9520266089590634,0.9053887225384184
+gemini-3-flash-preview,0.9609836737163503,0.9002256571212404,0.9033097691185296,0.9191046716179054,0.9134370407975112,91.59323924058683,0.24213953192956794,0.9311080653323691,0.9915530229672144,1.0,1926,1926,0.9271809248026928,0.8027728325705472,0.9776739356178609,0.8102768169906603,0.8157195789162777,0.8482630149822964,0.8679195129445441,0.8355447309584446,0.9590593887239334,0.8916353117350362
+claude-fable-5,0.9544882663119818,0.9084539374258854,0.9125656740745678,0.9253257401897927,0.9196281944238466,108.45349572194684,0.39993844235523723,0.9321674260051398,0.9779773029680953,1.0,1926,1926,0.9263795338237312,0.7986503096280622,0.9678089304257529,0.8369069573352913,0.8411428550072247,0.8676701069027709,0.882323289604615,0.8570108022124755,0.9511367212683994,0.9054905022057751
+gemini-3.5-flash,0.9553121943114703,0.8941350103289238,0.8995946946055895,0.911743621443697,0.9074634703451002,94.69485767184281,0.24849819683051586,0.9243805550021903,0.9862444928734878,1.0,1926,1926,0.9218369238486508,0.7736079030083943,0.9724818276220145,0.8014442434505356,0.8082979840178603,0.8324895766366438,0.8556964813203412,0.8244820713563452,0.9545621159815524,0.88063059556053
+deepseek-v4-pro,0.949162941454165,0.8934296925291699,0.8975455141155364,0.9179917619200594,0.9097925572366411,103.88830343855362,0.30025438338658783,0.9302032603817985,0.9802109464940681,1.0,1926,1926,0.9143830867498661,0.7872469200194228,0.9667705088265836,0.7968601383863302,0.8073069529034201,0.853071882677579,0.8770614475162618,0.8335751053708979,0.9493497983642315,0.8768878929741212
+grok-build-0.1,0.9454565099760769,0.8920834431837947,0.897448844040714,0.911717103441943,0.9056419609253753,110.5623943432874,0.3087931568220137,0.9213184530350494,0.9856680502360821,1.0,1926,1926,0.9129491065330215,0.7952873638530563,0.9626168224299065,0.8024693797747686,0.8140102031529518,0.8407252574419144,0.8562649086286446,0.8283674372495698,0.9440435221212979,0.865366363863269
+claude-sonnet-4.6,0.9444219305605858,0.8947118850481987,0.9001703356698266,0.9227650410541813,0.9119366244690852,75.75506348116517,0.3908815003272539,0.9300992361041344,0.9490938895443671,1.0,1926,1926,0.9070265577869125,0.8395570840845253,0.9517133956386293,0.8028755117411248,0.8140602090175635,0.8702654502266389,0.8790173914613146,0.8415546406116604,0.9465206182587071,0.906103500152286
+glm-5.3,0.9155338406596054,0.8710291090098558,0.8815353165924202,0.8927720554481405,0.8859285702861759,104.80508579499231,0.4226544967233401,0.8983778000942871,0.9333861713989211,0.9612866818346026,1926,1853,0.8981859337313037,0.7978972349112723,0.9293873312564901,0.8142414117551843,0.8327440086622091,0.8551520559397308,0.8684900450560394,0.8426568803532909,0.922440260767996,0.8750855562841544
+claude-opus-4.7,0.9289740231684763,0.8835917727786068,0.8885545337450622,0.8963163309926969,0.8943031545303557,123.41809169716089,0.5899509262930319,0.908749980605057,0.9345062340618155,1.0,1926,1926,0.8970828744027523,0.8083787040578894,0.9402907580477674,0.8089211219654718,0.8196380702473519,0.8379606645757669,0.8553430980287375,0.830465738704332,0.9343230410342837,0.8956716302565475
+gemini-3.5-flash-lite,0.9294245477893281,0.8950848050870075,0.8985934029706054,0.9041241477428685,0.9020559565280561,210.69852769549624,0.6709989663831847,0.9104214703117428,0.9600548342296911,1.0,1926,1926,0.895989047344456,0.6333355762826287,0.9454828660436138,0.8196638582078347,0.8272861024085786,0.8400487330497852,0.8558585918329278,0.8357143213747815,0.9341232308966709,0.8545120002122653
+gemini-3.1-flash-lite-preview,0.9330342644771338,0.8899651349459086,0.8976443905561986,0.9040012369345968,0.9005334982833954,189.91273850320692,0.6724012786849467,0.9105232306968775,0.9755600078477515,1.0,1926,1926,0.8893957280946369,0.7153540427140131,0.9527518172377986,0.7974947468007777,0.8161687728347259,0.8298683880789836,0.8429879613595438,0.8216299672685078,0.9336304661415563,0.8345295257490563
+grok-4.3,0.9363888967558768,0.8999223167893091,0.9082626059701155,0.9127411454702068,0.9104814642988877,205.35747437708156,0.6830590304353071,0.9209997889659193,0.9706701665167907,1.0,1926,1926,0.8888748947156832,0.6702726821783528,0.9485981308411215,0.8065004201160026,0.8262006614045171,0.8361956515529559,0.8560335490995266,0.8312325705432505,0.9367418171404236,0.8585391902081403
+qwen-3.7-max,0.9142488767323402,0.870320517390349,0.8755997124016868,0.8854344287329492,0.8805883394963456,226.4239219450457,0.5966462018712453,0.8909986994603973,0.9510034528727395,1.0,1926,1926,0.8676051203012336,0.707419880953127,0.9376947040498442,0.7737819823399782,0.7890178052878463,0.8122734152685972,0.8260077737447099,0.8002702441602828,0.9203146894518638,0.8148820385682011
+glm-5.2,0.8831572134726782,0.841240279005362,0.8460083082971677,0.8637357482720452,0.8555886718509442,115.30646939334679,0.6058035322051681,0.8713703518292016,0.9129817267467543,0.9263642066634998,1926,1790,0.8643016141206996,0.7282345044828232,0.8992731048805815,0.7689914527833324,0.7801297502310227,0.8246298909340085,0.8406347399597567,0.8035964584770301,0.8905286193116634,0.8366663119731662
+claude-haiku-5.5,0.9210274850193324,0.8924404110691503,0.8983600653293219,0.9034401145904055,0.9005041768053298,356.14759059802026,0.7143465617212607,0.9077761162324417,0.9640091011612457,1.0,1926,1926,0.8619453785574968,0.5245326935851915,0.9356178608515057,0.7957559516075987,0.8064807411206286,0.8191539260212687,0.8293276233854806,0.8126795605337442,0.9234023265299842,0.8164582019146276
+claude-opus-4.8,0.9174736903375664,0.8749226850713769,0.8793042988279907,0.8861567045412339,0.8838197799370083,155.1817475222531,0.6011118381240461,0.8948954313074317,0.9144716041047022,1.0,1926,1926,0.8464379142906836,0.7565887675605066,0.9288681204569055,0.761810405892832,0.7751485783293777,0.7895189221140895,0.8077935136059645,0.7835678549855659,0.9204442579758052,0.8499976429800123
+claude-haiku-4.5,0.8966026329098313,0.8614500961954722,0.8685503060111993,0.8741862529186979,0.8709516127982819,285.9620870400952,0.6711644214435234,0.8796197960677583,0.92324343989455,1.0,1926,1926,0.8319142941327162,0.6104053597313227,0.9179646936656283,0.7484030443059763,0.7640729188776659,0.7778065466863348,0.7930776791545334,0.7708400472561275,0.9014815413023568,0.7992431977175631
+qwen3.8-max,0.8977109788709101,0.8617098876473621,0.8647231379666469,0.8721432516059724,0.869201302795797,285.1587314514363,0.9677685207571491,0.8782289339632064,0.9346093562161663,1.0,1926,1926,0.8263743443074963,0.595255707169802,0.9158878504672897,0.7517280274378411,0.7577903638831444,0.7742293086902272,0.7861689848490023,0.7674791712150538,0.9020410653474253,0.7883835718862698
+minimax-m3,0.9103135173912139,0.8805877647645901,0.8871324190247615,0.8986857884568256,0.8917323029082989,370.75185980301586,0.8302798325756027,0.9005232393870184,0.961559287018209,1.0,1926,1926,0.8214746722945806,0.49701607900215683,0.9236760124610592,0.7578251350000993,0.7730803711082155,0.8006576968556125,0.8046371880694676,0.7840500977583487,0.9095185093657908,0.7618798790957588
+claude-sonnet-5,0.9154186825355719,0.8709162743450518,0.8779315804675009,0.8840399385616964,0.8809705729925791,202.59419596793757,0.7043032623559468,0.8909944985960673,0.9405477461607955,1.0,1926,1926,0.8210960142919488,0.7411117269472229,0.926791277258567,0.7269653065783305,0.7465347000453302,0.7592356728298055,0.7739034453124032,0.7516597811914673,0.9139461859957122,0.8134412464762122
+gpt-5.4-mini,0.8884117443269278,0.8557652247656322,0.8649087432841508,0.8667645861960281,0.8645991510564551,306.59198997809364,0.7750943992068712,0.8709580499800091,0.9179161871995511,1.0,1926,1926,0.8085718532822906,0.4783573747621803,0.9080996884735203,0.7362743640397311,0.7559761898009792,0.7594212915211596,0.7711644437346071,0.7557090722741192,0.899057003126912,0.763687089305864
+gpt-5.4-nano,0.8834166061863893,0.8429529800681144,0.8470634298035642,0.8551652816554159,0.852006642720915,368.3933263249559,0.832650542614489,0.8628448793565654,0.931785064965096,1.0,1926,1926,0.7602603596524262,0.4802707610012141,0.90134994807892,0.6562240607012416,0.6665421487411192,0.6904192923185751,0.7079498725066332,0.6802838435668923,0.8858833441659858,0.7165154190860602
+kimi-k2.6,0.7702883957563657,0.7596499370275689,0.7602226747137086,0.7640442526280284,0.7629228676706248,226.77877647749628,0.3694505742660025,0.7677746063131932,0.9047195412668892,0.7909245817156625,1926,1536,0.6944185476811513,0.2756890759947286,0.780373831775701,0.6716896626788399,0.6719051518296274,0.6820025706258881,0.6866435274779409,0.678060228153074,0.7553962463546361,0.621856826839531
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv
index 300a5cbb..55d7a144 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv
@@ -1,19 +1,19 @@
variable,mean_score,mean_exact,mean_within_1pct,mean_within_5pct,mean_threshold_score,mean_mae,mean_mape,mean_within_10pct,mean_accuracy,mean_coverage,total_n,parsed_n
-federal_income_tax_before_refundable_credits,0.8291135864963634,0.639515685195377,0.6568519537699504,0.7063841496973032,0.6834755090809026,746.0624693943573,0.5060190880264016,0.7311502476609796,,0.988167308750688,3634,3591
-federal_refundable_credits,0.9190875115986977,0.8684560780834072,0.8804347826086957,0.8926353149955635,0.8862577639751553,173.00560977562694,0.28203535783422007,0.9035048802129547,,0.9855811889973382,4508,4443
-free_school_meals_eligible,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.016597518325534345,,0.9795652173913044,0.9795652173913044,0.9960869565217392,4600,4582
-local_income_tax,0.9930434782608696,0.9930434782608696,0.9930434782608696,0.9930434782608696,0.9930434782608696,6.366352173913043,,0.9930434782608696,,0.9947826086956522,4600,4576
-payroll_tax,0.9512885899304093,0.8100686498855836,0.8423340961098399,0.8874141876430207,0.8620137299771166,261.8543115967522,0.04877540285534813,0.9082379862700228,,0.985812356979405,4370,4308
-person_chip_eligible,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.0355996940768357,,0.9579955784819455,0.9579955784819455,0.9934905428641612,8142,8089
-person_early_head_start_eligible,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.010885026903333539,,0.9834096109839817,0.9834096109839817,0.994279176201373,1748,1738
-person_head_start_eligible,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.02288329519450801,,0.9713958810068649,0.9713958810068649,0.994279176201373,1748,1738
-person_medicaid_eligible,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.07223360158487031,,0.9218396582055793,0.9218396582055793,0.9938426740387031,7958,7909
-person_medicare_eligible,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.02283453835564706,,0.9723205257836198,0.9723205257836198,0.9950707785642063,7912,7873
-person_wic_eligible,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.015484245550568322,,0.9792434291328912,0.9792434291328912,0.9947187423237535,8142,8099
-reduced_price_school_meals_eligible,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.01092008078387851,,0.9850000000000001,0.9850000000000001,0.9958695652173913,4600,4581
-self_employment_tax,0.976146812550527,0.9704347826086956,0.9752173913043478,0.9756521739130435,0.9744021739130434,23.08262975507091,0.037295638672444176,0.9763043478260869,,0.9882608695652174,4600,4546
-snap,0.8950815250162902,0.8419822346891072,0.8548387096774193,0.8740065451145395,0.8627279102384292,221.95803343535022,0.7373026265141042,0.8800841514726507,,0.9955586722767649,4278,4259
-ssi,0.9870536611205077,0.9773644105782161,0.9807261317794712,0.9836396234872254,0.9815665620797849,77.86850121885766,0.24877145637345424,0.9845360824742267,,0.9952935903182428,4462,4441
-state_income_tax_before_refundable_credits,0.843582367032528,0.6506393861892583,0.6736572890025575,0.7283887468030691,0.7044757033248082,204.39797151554535,0.38899262134225543,0.7652173913043478,,0.9905370843989769,3910,3873
-state_refundable_credits,0.854945577117676,0.8194321206743567,0.8198757763975155,0.8245341614906833,0.8230368234250223,61.134148706701154,0.947599145924913,0.8283052351375332,,0.9875776397515529,4508,4452
-tanf,0.9882009832576173,0.9845652173913043,0.9850000000000001,0.9850000000000001,0.9850000000000001,43.13453546934715,0.6283439336657944,0.9854347826086957,,0.9952173913043478,4600,4578
+federal_income_tax_before_refundable_credits,0.8260348842264251,0.60612968591692,0.6289260385005065,0.6841438703140831,0.6582446808510638,950.7248119174031,0.4602640168706383,0.7137791286727457,,0.9875886524822695,3948,3899
+federal_refundable_credits,0.918452713087642,0.868866695614416,0.8805905340859748,0.8925314806773773,0.8862896222318715,182.4079877305546,0.29702746009548586,0.9031697785497177,,0.9858879722101607,4606,4541
+free_school_meals_eligible,0.9797872340425531,0.9797872340425531,0.9797872340425531,0.9797872340425531,0.9797872340425531,0.016457145595203827,,0.9797872340425531,0.9797872340425531,0.9961702127659574,4700,4682
+local_income_tax,0.9926183239253148,0.9926183239253148,0.9926183239253148,0.9926183239253148,0.9926183239253148,8.095797113629255,,0.9926183239253148,,0.9947894051237516,4606,4582
+payroll_tax,0.9479575129406436,0.7994237588652482,0.8315602836879433,0.8792109929078014,0.8538342198581561,293.480137756744,0.054074318027316305,0.9051418439716311,,0.9858156028368795,4512,4448
+person_chip_eligible,0.9585286693112154,0.9585286693112154,0.9585286693112154,0.9585286693112154,0.9585286693112154,0.03520287404418755,,0.9585286693112154,0.9585286693112154,0.9936290419521577,8319,8266
+person_early_head_start_eligible,0.9837625979843225,0.9837625979843225,0.9837625979843225,0.9837625979843225,0.9837625979843225,0.010653430586241335,,0.9837625979843225,0.9837625979843225,0.9944008958566629,1786,1776
+person_head_start_eligible,0.9697648376259799,0.9697648376259799,0.9697648376259799,0.9697648376259799,0.9697648376259799,0.024636058230683093,,0.9697648376259799,0.9697648376259799,0.9944008958566629,1786,1776
+person_medicaid_eligible,0.9212888943549379,0.9212888943549379,0.9212888943549379,0.9212888943549379,0.9212888943549379,0.07291046629103405,,0.9212888943549379,0.9212888943549379,0.9939736809740498,8131,8082
+person_medicare_eligible,0.9722909450766947,0.9722909450766947,0.9722909450766947,0.9722909450766947,0.9722909450766947,0.022967202804289898,,0.9722909450766947,0.9722909450766947,0.9951756556160317,8084,8045
+person_wic_eligible,0.9792042312777979,0.9792042312777979,0.9792042312777979,0.9792042312777979,0.9792042312777979,0.01563562054005617,,0.9792042312777979,0.9792042312777979,0.9948311095083544,8319,8276
+reduced_price_school_meals_eligible,0.9853191489361702,0.9853191489361702,0.9853191489361702,0.9853191489361702,0.9853191489361702,0.01068773863954067,,0.9853191489361702,0.9853191489361702,0.9959574468085107,4700,4681
+self_employment_tax,0.9766513855145351,0.9702127659574469,0.9757446808510638,0.9761702127659575,0.9747340425531915,22.594977967251086,0.036538885227576444,0.9768085106382978,,0.9885106382978723,4700,4646
+snap,0.894453885168825,0.8423701670098376,0.8549530999771219,0.8737131091283459,0.8626744452070465,228.33504164640317,0.7420749633423726,0.8796614047128803,,0.9956531686113018,4371,4352
+ssi,0.9868904224434537,0.9774073261680193,0.980697521386269,0.9835490239087519,0.9815200701908313,80.585490773973,0.26475504240806164,0.9844264093002851,,0.9953937266944505,4559,4538
+state_income_tax_before_refundable_credits,0.8438015943726606,0.6395206651993153,0.6629982880899975,0.7185130838835901,0.6964416727806308,295.257439916886,0.3765259988537218,0.7647346539496208,,0.9892394228417706,4089,4045
+state_refundable_credits,0.8622144562901147,0.8280503690838037,0.8284845853234911,0.8328267477203648,0.8315240990013028,66.47487465905856,0.7182028543412085,0.8367346938775511,,0.9878419452887538,4606,4550
+tanf,0.9882392602095829,0.9846808510638299,0.9851063829787233,0.9851063829787233,0.9851063829787233,43.50719620071742,0.636423413368712,0.985531914893617,,0.9953191489361702,4700,4678
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv
index 2fc00d90..9406d6f9 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv
@@ -2,6 +2,7 @@ model,total_rows,parsed_rows,error_rows,total_provider_reported_cost_usd,total_r
claude-fable-5,1984,1984,0,,,,0,,,,,,,,,,,
claude-fable-5.1,1984,1984,0,,25.698449999999973,25.698449999999973,0,25.698449999999973,5614.7681494661665,1398.6418070793152,1788307560.546109,1788308959.187916,208620.0,472245.0,680865.0,0.0,0.0,0.0
claude-haiku-4.5,1984,1984,16,,5.54126899999997,5.54126899999997,1984,5.54126899999997,5927.419744440027,,,,3409714.0,426311.0,3836025.0,0.0,0.0,
+claude-haiku-5.5,1984,1984,0,,0.1298898,0.1298898,0,0.1298898,550.8162037961156,322.6581311225891,1791495684.701819,1791496007.35995,624268.0,134926.0,759194.0,0.0,0.0,0.0
claude-opus-4.7,1984,1984,0,,29.30251999999998,29.30251999999998,1984,29.30251999999998,6206.862578470866,,,,4246299.0,322841.0,4569140.0,0.0,0.0,
claude-opus-4.8,1984,1984,0,,25.505849999999974,25.505849999999974,1984,25.505849999999974,6341.968646365076,,,,3452460.0,329742.0,3782202.0,0.0,0.0,
claude-opus-5,1984,1984,0,,6.748584999999974,6.748584999999974,1984,6.748584999999974,1808.0877771629603,516.2362098693848,1784918536.916325,1784919053.152535,595917.0,150760.0,746677.0,0.0,0.0,0.0
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz
index 3bc51085..7a277495 100644
Binary files a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz and b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz differ
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz
index 7bfbb864..c057897c 100644
Binary files a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz and b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz differ
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json
index 718d38e5..f18ee2e9 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"rule": "An output is excluded from scoring for every model when its reference depends on an input or definition that the certified household data never carried and the prompt therefore never stated, and a careful reader could take the stated facts the other way. Excluded outputs stay in the published payload, marked scored=false, so every prompt and response remains inspectable.",
- "derivation": "All 1,984 references were recomputed with policyengine-us 1.755.4 (the version that produced them; base recompute matched the frozen CSV on every row) under each alternative reading; every output whose value moved is listed. 2026-09-05. On 2026-10-05 eight more outputs were excluded on review of release dashboard-data-20260930 (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll); their records are computed on policyengine-us 2.15.17, whose values are the references, and each sweep's baseline reproduced every scored reference. On 2026-09-29 the references moved to policyengine-us 2.15.17 with the pre-freeze conventions. Every excluded output was recomputed there, and each one that moved was re-reviewed and stays excluded (the reference sidecar's engine_upgrade revision lists them). Records decided before that date keep the values they were decided on; the four records decided that day were computed on 2.15.17. The audit excluded one of them on review of the release. Its reference did not move, so the engine_upgrade revision does not list it (final_actions.json audit_exclusions).",
+ "derivation": "All 1,984 references were recomputed with policyengine-us 1.755.4 (the version that produced them; base recompute matched the frozen CSV on every row) under each alternative reading; every output whose value moved is listed. 2026-09-05. On 2026-10-05 eight more outputs were excluded on review of release dashboard-data-20260930 (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll); their records are computed on policyengine-us 2.15.17, whose values are the references, and each sweep's baseline reproduced every scored reference. On 2026-10-06 ten more outputs were excluded on review of release dashboard-data-20261006 (reference_audit/2026-10-05-reference-adversary and reference_audit/2026-10-05-louisiana); their records are computed on policyengine-us 2.15.17, whose values are the references. On 2026-09-29 the references moved to policyengine-us 2.15.17 with the pre-freeze conventions. Every excluded output was recomputed there, and each one that moved was re-reviewed and stays excluded (the reference sidecar's engine_upgrade revision lists them). Records decided before that date keep the values they were decided on; the four records decided that day were computed on 2.15.17. The audit excluded one of them on review of the release. Its reference did not move, so the engine_upgrade revision does not list it (final_actions.json audit_exclusions). On 2026-10-10 the references moved from policyengine-us 2.15.17 to policyengine-us 2.38.6 with the same pre-freeze conventions (reference_audit/2026-10-09-engine-upgrade), and every output was recomputed there. The new engine fixes the defects behind 18 excluded outputs, each landing within $1 of its audited corrected value, so their records were removed and the outputs are scored again (the reference sidecar's second engine_upgrade revision lists them under regenerated_exclusions). Two scored outputs the new engine moved were excluded that day; their records are computed on 2.38.6. Of the excluded outputs that stay excluded, 17 moved; each was re-reviewed and keeps the value it was decided on (excluded_outputs_rechecked).",
"exclusions": [
{
"scenario_id": "scenario_007",
@@ -170,22 +170,6 @@
"unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable",
"note": "Corrected value applies r24_disability_benefits_taxable with every publication convention and upstream fix."
},
- {
- "scenario_id": "scenario_003",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r01_ira_compensation",
- "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
- "frozen_value": 22154.69921875,
- "alternative_value": 22400.654297,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
- "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "upstream": "to be filed",
- "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_005",
"variable": "federal_income_tax_before_refundable_credits",
@@ -202,22 +186,6 @@
"upstream": "to be filed",
"note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input)."
},
- {
- "scenario_id": "scenario_005",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g+r11_ca_itemized_conformity",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.",
- "frozen_value": 41051.51171875,
- "alternative_value": 40920.402344,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076",
- "upstream": "to be filed",
- "note": "Corrected value applies r02_ira_219g_v2 and r11_ca_itemized_conformity_v2 together with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_007",
"variable": "state_income_tax_before_refundable_credits",
@@ -234,22 +202,6 @@
"upstream": "to be filed",
"note": "Corrected value applies r07_idaho_health_premiums_v2 with every publication convention and upstream fix."
},
- {
- "scenario_id": "scenario_008",
- "variable": "payroll_tax",
- "reason_code": "reference_engine_defect",
- "root_cause": "r05_nj_worker_ui",
- "alternative_reading": "New Jersey workers contribute 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds, up to the 2026 wage base of $44,800, alongside the TDI and FLI contributions.",
- "frozen_value": 2162.760009765625,
- "alternative_value": 2276.659912,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax",
- "law": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates",
- "upstream": "to be filed",
- "note": "Corrected value applies r05_nj_worker_ui_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_020",
"variable": "federal_income_tax_before_refundable_credits",
@@ -263,54 +215,6 @@
"unlisted_input": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say",
"note": "Corrected value applies r25_niit_excluded with every publication convention and upstream fix."
},
- {
- "scenario_id": "scenario_022",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r11_ca_itemized_conformity",
- "alternative_reading": "California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.",
- "frozen_value": 2439.650146484375,
- "alternative_value": 1968.80542,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions",
- "law": "Cal. R&TC 17024.5 (conformity date), 17076",
- "upstream": "to be filed",
- "note": "Corrected value applies r11_ca_itemized_conformity_v2 with every publication convention and upstream fix."
- },
- {
- "scenario_id": "scenario_039",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r03_estate_income",
- "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
- "frozen_value": 1345.510498046875,
- "alternative_value": 8596.02832,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
- "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "upstream": "related: PolicyEngine/policyengine-us#9304",
- "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix."
- },
- {
- "scenario_id": "scenario_039",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r03_estate_income",
- "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
- "frozen_value": 514.4984130859375,
- "alternative_value": 1975.798218,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
- "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "upstream": "related: PolicyEngine/policyengine-us#9304",
- "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_042",
"variable": "state_income_tax_before_refundable_credits",
@@ -327,22 +231,6 @@
"upstream": "to be filed",
"note": "Corrected value applies r06_wi_act15_before_refundable_v2 and r32_wi_capital_gain_distributions together with every publication convention and upstream fix. The output also moves under r04_capital_gain_distributions (engine defect, fixed upstream)."
},
- {
- "scenario_id": "scenario_049",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
- "frozen_value": 30543.908203125,
- "alternative_value": 30702.589844,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "upstream": "to be filed",
- "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_052",
"variable": "federal_income_tax_before_refundable_credits",
@@ -427,38 +315,6 @@
"unlisted_input": "the relationship of an adult tax dependent to the filers who claim them",
"note": "Corrected value applies r067_adult_dependent_nonchild with every publication convention and upstream fix."
},
- {
- "scenario_id": "scenario_064",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r01_ira_compensation",
- "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
- "frozen_value": 4439.28955078125,
- "alternative_value": 4441.455078,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
- "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "upstream": "to be filed",
- "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
- },
- {
- "scenario_id": "scenario_064",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r01_ira_compensation",
- "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
- "frozen_value": 4605.9970703125,
- "alternative_value": 4599.621582,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
- "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
- "upstream": "to be filed",
- "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_067",
"variable": "dependent1_medicaid_eligible",
@@ -504,22 +360,6 @@
"upstream": "to be filed",
"note": "Corrected value applies ma_part_a_loss_offset with every publication convention and upstream fix. The output also moves under r23_ma_interest_source (unlisted input)."
},
- {
- "scenario_id": "scenario_082",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
- "frozen_value": 9563.052734375,
- "alternative_value": 9564.916992,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "upstream": "to be filed",
- "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_091",
"variable": "state_income_tax_before_refundable_credits",
@@ -536,22 +376,6 @@
"upstream": "to be filed",
"note": "Corrected value applies r32_wi_capital_gain_distributions with every publication convention and upstream fix. The output also moves under r04_capital_gain_distributions (engine defect, fixed upstream)."
},
- {
- "scenario_id": "scenario_099",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
- "frozen_value": 10679.75,
- "alternative_value": 10711.484375,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "upstream": "to be filed",
- "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_099",
"variable": "state_income_tax_before_refundable_credits",
@@ -626,38 +450,6 @@
"unlisted_input": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits",
"note": "Corrected value applies r10_wi_homestead_income and r16_survivor_benefits_federal_v2 together with every publication convention and upstream fix."
},
- {
- "scenario_id": "scenario_110",
- "variable": "federal_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g+r03_estate_income",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
- "frozen_value": 23897.443359375,
- "alternative_value": 25747.773438,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "upstream": "related: PolicyEngine/policyengine-us#9304",
- "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix."
- },
- {
- "scenario_id": "scenario_110",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g+r03_estate_income",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
- "frozen_value": 4057.47216796875,
- "alternative_value": 4236.807617,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
- "upstream": "related: PolicyEngine/policyengine-us#9304",
- "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_112",
"variable": "snap",
@@ -732,22 +524,6 @@
"upstream": "to be filed",
"note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input). The output also moves under the state income tax withheld reading (unlisted input): the reference's SALT uses the Connecticut withholding estimate ($12,010.05); with the engine's own liability ($11,917.18) the 2.15.17 value is $40,438.95, and with no state income tax paid it is $43,619.93. With r02's IRA fix on 2.15.17 the value is $40,423.18 under the withholding estimate, $40,445.46 under the liability and $43,629.21 with no state income tax paid. policyengine-us 2.15.17 also leaves the household's Connecticut paid leave contributions ($827.98) out of SALT (Rev. Rul. 2025-4); with them the 2.15.17 value is $40,217.95."
},
- {
- "scenario_id": "scenario_120",
- "variable": "state_income_tax_before_refundable_credits",
- "reason_code": "reference_engine_defect",
- "root_cause": "r02_ira_219g",
- "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
- "frozen_value": 11917.18359375,
- "alternative_value": 11919.057617,
- "engine_version": "policyengine-us 1.755.4",
- "decided_on": "2026-09-22",
- "decided_by": "developer",
- "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
- "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
- "upstream": "to be filed",
- "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
- },
{
"scenario_id": "scenario_121",
"variable": "federal_income_tax_before_refundable_credits",
@@ -917,6 +693,128 @@
"unlisted_input": "Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either",
"note": "Found 2026-10-05 by sweeping every reference with each person's Medicare Part B premium at 0, and again with Medicare enrollment off (reference_audit/2026-10-05-medicare-part-b/scripts/sweep_part_b.py, on policyengine-us 2.15.17 with latest_final; its baseline reproduces all 1,928 scored references). Of 39 Medicare-eligible people in 33 households, the engine charges the premium in 30 (Medicare Savings Program coverage pays it for the 4 eligible people in the other 3); only this household's two tax outputs move, under either reading. The federal medical expense deduction is $4,441.86 in the reference ($12,434.80 of expenses less 7.5% of $106,572.56 AGI) and $2,007.06 without the premium; the household itemizes under both. The output also depends on the state income tax in the federal SALT deduction, which PolicyEngine/policybench#191 proposes to treat as an unlisted input (decision d963), through the federal itemization election, which Virginia follows (va_deductions reads tax_unit_itemizes). Without the premium the output is $3,654.15 under both the withholding estimate and the liability; with no state income tax paid it is $4,127.67, with or without the local sales tax estimate, because the household then takes the federal and Virginia standard deductions. With the premium it is $3,514.15 under all four readings. The premium is the standard premium: the engine reads the income-related adjustment from MAGI two years earlier, which no household sets, and at this household's 2026 MAGI ($106,572.56) none would apply. GPT-6 Astra ($3,654.16) and GPT-6.1 Sol ($3,654.66) answered within $1 of the alternative value, and no model within $1 of the frozen value."
},
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_law_published_after_freeze",
+ "alternative_reading": "R.S. 47:294(B) multiplies 2025's $12,500 by the calendar-2025 CPI-U increase but sets no rounding rule and gives the Department no duty to publish the result. Before the 2026-07-03 freeze Louisiana published only its withholding and estimated-tax amount, $12,875, and said the amount allowed on 2026 returns would differ; its first 2026 return amount, $12,838, came in RIB 26-019 on 2026-09-28, after the freeze. policyengine-us carries $12,835, which PolicyEngine/policyengine-us#8411 computed on 2026-05-24 from published CPI-U values ($12,500 x 324.054 / 315.605, rounded to the dollar): a computation under the statute, not a forecast, but a figure no Louisiana publication stated before the freeze. The reference is 3% of this single filer's Louisiana AGI ($40,180) less $12,835. Both Louisiana outputs that rest on the amount are excluded, and the published-amounts convention stays in force for every other reference (decision d994).",
+ "frozen_value": 820.3499755859375,
+ "alternative_value": 820.260009765625,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d994",
+ "root_cause": "la_2026_standard_deduction_unpublished",
+ "published": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "law": "La. R.S. 47:294(A)-(B) (Act 11 of the 2024 Third Extraordinary Session); BLS CPI-U, series CUUR0000SA0, December 2024 and December 2025 (released 2026-01-13); Louisiana Department of Revenue, Revenue Information Bulletin 26-019 (2026-09-28)",
+ "note": "Found 2026-10-05 while preparing an interview on the board, and audited in reference_audit/2026-10-05-louisiana (PolicyEngine/policybench#192): the sweep (scripts/sweep_la_standard_deduction.py, policyengine-us 2.15.17 with latest_final) reproduces all 1,928 scored references at $12,835 and moves only these two outputs under every candidate amount (verification/sweep_la_standard_deduction.log). Values under the candidates: $12,500 (2025's amount, the last return amount published) $830.40; $12,875 (the withholding amount) $819.15; $12,838 (RIB 26-019) $820.26. Of release dashboard-data-20260930's 46 models, none answered within $1 of the frozen value or of $820.26, and 21 answered within $1 of $830.40 (verification/leaderboard_impact.json)."
+ },
+ {
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_law_published_after_freeze",
+ "alternative_reading": "R.S. 47:294(B) multiplies 2025's $12,500 by the calendar-2025 CPI-U increase but sets no rounding rule and gives the Department no duty to publish the result. Before the 2026-07-03 freeze Louisiana published only its withholding and estimated-tax amount, $12,875, and said the amount allowed on 2026 returns would differ; its first 2026 return amount, $12,838, came in RIB 26-019 on 2026-09-28, after the freeze. policyengine-us carries $12,835, which PolicyEngine/policyengine-us#8411 computed on 2026-05-24 from published CPI-U values ($12,500 x 324.054 / 315.605, rounded to the dollar): a computation under the statute, not a forecast, but a figure no Louisiana publication stated before the freeze. The reference is 3% of this single filer's Louisiana AGI ($23,006.18) less $12,835. Both Louisiana outputs that rest on the amount are excluded, and the published-amounts convention stays in force for every other reference (decision d994).",
+ "frozen_value": 305.13525390625,
+ "alternative_value": 305.0452575683594,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d994",
+ "root_cause": "la_2026_standard_deduction_unpublished",
+ "published": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "law": "La. R.S. 47:294(A)-(B) (Act 11 of the 2024 Third Extraordinary Session); BLS CPI-U, series CUUR0000SA0, December 2024 and December 2025 (released 2026-01-13); Louisiana Department of Revenue, Revenue Information Bulletin 26-019 (2026-09-28)",
+ "note": "Found 2026-10-05 while preparing an interview on the board, and audited in reference_audit/2026-10-05-louisiana (PolicyEngine/policybench#192): the sweep (scripts/sweep_la_standard_deduction.py, policyengine-us 2.15.17 with latest_final) reproduces all 1,928 scored references at $12,835 and moves only these two outputs under every candidate amount (verification/sweep_la_standard_deduction.log). Values under the candidates: $12,500 (2025's amount, the last return amount published) $315.19; $12,875 (the withholding amount) $303.94; $12,838 (RIB 26-019) $305.05. Of release dashboard-data-20260930's 46 models, none answered within $1 of the frozen value or of $305.05, and 19 answered within $1 of $315.19 (verification/leaderboard_impact.json)."
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen_value": 8628.990234375,
+ "alternative_value": 11848.990234375,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "72 P.S. 7302(a); 2025 PA-40 instructions p. 3 (a minor with PA income over $33 must file); 26 U.S.C. 73(a); RSMo 143.011",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_093__federal_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/pa_123.md; verification/definition_conformance.md (household scope)). Alternative value: head and spouse return plus the dependent's own federal return ($3,220.00), verification/definition_conformance.json.",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "root_cause": "household_scope_dependent_returns",
+ "unlisted_input": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "alternative_reading": "A dependent with $45,000 of wages must file their own federal and state return. The engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one 'Tax unit' block, and the request asks for quantities 'for this household' and says to assume filing when required. Careful readers split both ways (pa_123.md section 3)."
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen_value": 3388.24560546875,
+ "alternative_value": 4528.07861328125,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "72 P.S. 7302(a); 2025 PA-40 instructions p. 3 (a minor with PA income over $33 must file); 26 U.S.C. 73(a); RSMo 143.011",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_093__state_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/pa_123.md; verification/definition_conformance.md (household scope)). Alternative value: head and spouse return plus the dependent's own MO-1040, verification/definition_conformance.json.",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "root_cause": "household_scope_dependent_returns",
+ "unlisted_input": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "alternative_reading": "A dependent with $45,000 of wages must file their own federal and state return. The engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one 'Tax unit' block, and the request asks for quantities 'for this household' and says to assume filing when required. Careful readers split both ways (pa_123.md section 3)."
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen_value": 5200.240234375,
+ "alternative_value": 8420.240234375,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "72 P.S. 7302(a); 2025 PA-40 instructions p. 3 (a minor with PA income over $33 must file); 26 U.S.C. 73(a); RSMo 143.011",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_123__federal_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/pa_123.md; verification/definition_conformance.md (household scope)). Alternative value: head and spouse return plus the child's own federal return ($3,220.00), verification/definition_conformance.json.",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "root_cause": "household_scope_dependent_returns",
+ "unlisted_input": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "alternative_reading": "A dependent with $45,000 of wages must file their own federal and state return. The engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one 'Tax unit' block, and the request asks for quantities 'for this household' and says to assume filing when required. Careful readers split both ways (pa_123.md section 3)."
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen_value": 3070.061279296875,
+ "alternative_value": 4451.561279296875,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "72 P.S. 7302(a); 2025 PA-40 instructions p. 3 (a minor with PA income over $33 must file); 26 U.S.C. 73(a); RSMo 143.011",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_123__state_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/pa_123.md; verification/definition_conformance.md (household scope)). Alternative value: head and spouse return plus the child's own PA-40 ($1,381.50), verification/definition_conformance.json.",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "root_cause": "household_scope_dependent_returns",
+ "unlisted_input": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
+ "alternative_reading": "A dependent with $45,000 of wages must file their own federal and state return. The engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one 'Tax unit' block, and the request asks for quantities 'for this household' and says to assume filing when required. Careful readers split both ways (pa_123.md section 3)."
+ },
+ {
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "unlisted_input": "county of residence (Indiana county income tax)",
+ "alternative_reading": "The prompt places the household in Indiana but states no county, and Indiana's county income taxes (IC 6-3.6) differ by county. policyengine-us 2.38.6 computes Indiana county tax (in_county_tax) in local_income_tax at the rate of the county it assigns a household with no stated county (Adams County on policyengine-us 2.38.6). Read without a county, as policyengine-us 2.15.17 did, the household owes no county tax.",
+ "frozen_value": 670.5501098632812,
+ "alternative_value": 0.0,
+ "engine_version": "policyengine-us 2.38.6",
+ "decided_on": "2026-10-10",
+ "decided_by": "developer",
+ "note": "Found in the 2026-10-10 engine upgrade (the US hub's sweep, cells_pe.md: local-tax standardization wires Indiana county income tax into local_income_tax). The alternative is the policyengine-us 2.15.17 reference."
+ },
+ {
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "unlisted_input": "county of residence (Indiana county income tax)",
+ "alternative_reading": "The prompt places the household in Indiana but states no county, and Indiana's county income taxes (IC 6-3.6) differ by county. policyengine-us 2.38.6 computes Indiana county tax (in_county_tax) in local_income_tax at the rate of the county it assigns a household with no stated county (Adams County on policyengine-us 2.38.6). Read without a county, as policyengine-us 2.15.17 did, the household owes no county tax.",
+ "frozen_value": 1348.83203125,
+ "alternative_value": 0.0,
+ "engine_version": "policyengine-us 2.38.6",
+ "decided_on": "2026-10-10",
+ "decided_by": "developer",
+ "note": "Found in the 2026-10-10 engine upgrade (the US hub's sweep, cells_pe.md: local-tax standardization wires Indiana county income tax into local_income_tax). The alternative is the policyengine-us 2.15.17 reference."
+ },
{
"scenario_id": "scenario_023",
"variable": "head_medicaid_eligible",
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv
index 67d4e5ae..323a4522 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv
@@ -55,7 +55,7 @@ scenario_002,head_medicare_eligible,1.0,5285.2001953125
scenario_002,spouse_medicare_eligible,1.0,5285.2001953125
scenario_002,free_school_meals_eligible,0.0,0.0
scenario_002,reduced_price_school_meals_eligible,0.0,0.0
-scenario_003,federal_income_tax_before_refundable_credits,22154.69921875,
+scenario_003,federal_income_tax_before_refundable_credits,22400.654296875,
scenario_003,federal_refundable_credits,0.0,
scenario_003,payroll_tax,0.0,
scenario_003,self_employment_tax,0.0,
@@ -99,7 +99,7 @@ scenario_005,federal_income_tax_before_refundable_credits,106505.8984375,
scenario_005,federal_refundable_credits,0.0,
scenario_005,payroll_tax,36044.0,
scenario_005,self_employment_tax,0.0,
-scenario_005,state_income_tax_before_refundable_credits,41051.51171875,
+scenario_005,state_income_tax_before_refundable_credits,40975.25390625,
scenario_005,state_refundable_credits,0.0,
scenario_005,local_income_tax,0.0,
scenario_005,snap,0.0,
@@ -133,7 +133,7 @@ scenario_007,free_school_meals_eligible,0.0,0.0
scenario_007,reduced_price_school_meals_eligible,0.0,0.0
scenario_008,federal_income_tax_before_refundable_credits,0.0,
scenario_008,federal_refundable_credits,12433.611328125,
-scenario_008,payroll_tax,2162.760009765625,
+scenario_008,payroll_tax,2276.659912109375,
scenario_008,self_employment_tax,565.1820068359375,
scenario_008,state_income_tax_before_refundable_credits,264.80999755859375,
scenario_008,state_refundable_credits,5842.39990234375,
@@ -275,7 +275,7 @@ scenario_015,payroll_tax,3442.5,
scenario_015,self_employment_tax,0.0,
scenario_015,state_income_tax_before_refundable_credits,1236.3267822265625,
scenario_015,state_refundable_credits,0.0,
-scenario_015,local_income_tax,0.0,
+scenario_015,local_income_tax,670.5501098632812,
scenario_015,snap,0.0,
scenario_015,ssi,0.0,
scenario_015,tanf,0.0,
@@ -305,7 +305,7 @@ scenario_018,federal_income_tax_before_refundable_credits,5211.05078125,
scenario_018,federal_refundable_credits,0.0,
scenario_018,payroll_tax,5202.38232421875,
scenario_018,self_employment_tax,0.0,
-scenario_018,state_income_tax_before_refundable_credits,1146.05224609375,
+scenario_018,state_income_tax_before_refundable_credits,1137.30224609375,
scenario_018,state_refundable_credits,0.0,
scenario_018,local_income_tax,0.0,
scenario_018,snap,0.0,
@@ -353,7 +353,7 @@ scenario_022,federal_income_tax_before_refundable_credits,11113.5703125,
scenario_022,federal_refundable_credits,0.0,
scenario_022,payroll_tax,4710.83251953125,
scenario_022,self_employment_tax,0.0,
-scenario_022,state_income_tax_before_refundable_credits,2439.650146484375,
+scenario_022,state_income_tax_before_refundable_credits,1968.805419921875,
scenario_022,state_refundable_credits,0.0,
scenario_022,local_income_tax,0.0,
scenario_022,snap,0.0,
@@ -385,7 +385,7 @@ scenario_025,federal_income_tax_before_refundable_credits,7031.03564453125,
scenario_025,federal_refundable_credits,0.0,
scenario_025,payroll_tax,4798.48486328125,
scenario_025,self_employment_tax,0.0,
-scenario_025,state_income_tax_before_refundable_credits,1921.57275390625,
+scenario_025,state_income_tax_before_refundable_credits,1916.606201171875,
scenario_025,state_refundable_credits,0.0,
scenario_025,local_income_tax,0.0,
scenario_025,snap,0.0,
@@ -651,11 +651,11 @@ scenario_038,child1_early_head_start_eligible,0.0,0.0
scenario_038,child2_early_head_start_eligible,0.0,0.0
scenario_038,free_school_meals_eligible,1.0,2261.924560546875
scenario_038,reduced_price_school_meals_eligible,0.0,0.0
-scenario_039,federal_income_tax_before_refundable_credits,1345.510498046875,
+scenario_039,federal_income_tax_before_refundable_credits,8596.0283203125,
scenario_039,federal_refundable_credits,0.0,
scenario_039,payroll_tax,0.0,
scenario_039,self_employment_tax,0.0,
-scenario_039,state_income_tax_before_refundable_credits,514.4984130859375,
+scenario_039,state_income_tax_before_refundable_credits,1975.7982177734375,
scenario_039,state_refundable_credits,0.0,
scenario_039,local_income_tax,0.0,
scenario_039,snap,0.0,
@@ -708,7 +708,7 @@ scenario_043,federal_refundable_credits,0.0,
scenario_043,payroll_tax,330.7310791015625,
scenario_043,self_employment_tax,0.0,
scenario_043,state_income_tax_before_refundable_credits,0.0,
-scenario_043,state_refundable_credits,19.0,
+scenario_043,state_refundable_credits,0.0,
scenario_043,local_income_tax,0.0,
scenario_043,snap,3576.0,
scenario_043,ssi,0.0,
@@ -803,7 +803,7 @@ scenario_048,head_chip_eligible,0.0,0.0
scenario_048,head_medicare_eligible,1.0,5285.2001953125
scenario_048,free_school_meals_eligible,0.0,0.0
scenario_048,reduced_price_school_meals_eligible,0.0,0.0
-scenario_049,federal_income_tax_before_refundable_credits,30543.908203125,
+scenario_049,federal_income_tax_before_refundable_credits,30702.58984375,
scenario_049,federal_refundable_credits,0.0,
scenario_049,payroll_tax,15134.5,
scenario_049,self_employment_tax,0.0,
@@ -1007,11 +1007,11 @@ scenario_062,head_chip_eligible,0.0,0.0
scenario_062,head_medicare_eligible,1.0,5285.2001953125
scenario_062,free_school_meals_eligible,0.0,0.0
scenario_062,reduced_price_school_meals_eligible,0.0,0.0
-scenario_064,federal_income_tax_before_refundable_credits,4439.28955078125,
+scenario_064,federal_income_tax_before_refundable_credits,4441.455078125,
scenario_064,federal_refundable_credits,0.0,
scenario_064,payroll_tax,7443.0673828125,
scenario_064,self_employment_tax,0.0,
-scenario_064,state_income_tax_before_refundable_credits,4605.9970703125,
+scenario_064,state_income_tax_before_refundable_credits,4599.62158203125,
scenario_064,state_refundable_credits,0.0,
scenario_064,local_income_tax,0.0,
scenario_064,snap,0.0,
@@ -1063,7 +1063,7 @@ scenario_067,payroll_tax,6300.5400390625,
scenario_067,self_employment_tax,0.0,
scenario_067,state_income_tax_before_refundable_credits,2486.908935546875,
scenario_067,state_refundable_credits,0.0,
-scenario_067,local_income_tax,0.0,
+scenario_067,local_income_tax,1348.83203125,
scenario_067,snap,0.0,
scenario_067,ssi,0.0,
scenario_067,tanf,0.0,
@@ -1201,7 +1201,7 @@ scenario_076,federal_income_tax_before_refundable_credits,19611.80078125,
scenario_076,federal_refundable_credits,0.0,
scenario_076,payroll_tax,12240.0,
scenario_076,self_employment_tax,0.0,
-scenario_076,state_income_tax_before_refundable_credits,6818.34423828125,
+scenario_076,state_income_tax_before_refundable_credits,6828.34423828125,
scenario_076,state_refundable_credits,465.0,
scenario_076,local_income_tax,0.0,
scenario_076,snap,0.0,
@@ -1309,12 +1309,12 @@ scenario_081,head_chip_eligible,0.0,0.0
scenario_081,head_medicare_eligible,0.0,0.0
scenario_081,free_school_meals_eligible,0.0,0.0
scenario_081,reduced_price_school_meals_eligible,0.0,0.0
-scenario_082,federal_income_tax_before_refundable_credits,9563.052734375,
+scenario_082,federal_income_tax_before_refundable_credits,9579.767578125,
scenario_082,federal_refundable_credits,0.0,
scenario_082,payroll_tax,8108.02734375,
scenario_082,self_employment_tax,0.0,
-scenario_082,state_income_tax_before_refundable_credits,5598.55224609375,
-scenario_082,state_refundable_credits,667.0,
+scenario_082,state_income_tax_before_refundable_credits,5599.1435546875,
+scenario_082,state_refundable_credits,1187.5478515625,
scenario_082,local_income_tax,0.0,
scenario_082,snap,0.0,
scenario_082,ssi,0.0,
@@ -1363,7 +1363,7 @@ scenario_084,head_chip_eligible,0.0,0.0
scenario_084,head_medicare_eligible,0.0,0.0
scenario_084,free_school_meals_eligible,0.0,0.0
scenario_084,reduced_price_school_meals_eligible,0.0,0.0
-scenario_085,federal_income_tax_before_refundable_credits,2322.760009765625,
+scenario_085,federal_income_tax_before_refundable_credits,2322.8486328125,
scenario_085,federal_refundable_credits,0.0,
scenario_085,payroll_tax,0.0,
scenario_085,self_employment_tax,0.0,
@@ -1541,7 +1541,7 @@ scenario_098,head_chip_eligible,0.0,0.0
scenario_098,head_medicare_eligible,1.0,5285.2001953125
scenario_098,free_school_meals_eligible,0.0,0.0
scenario_098,reduced_price_school_meals_eligible,0.0,0.0
-scenario_099,federal_income_tax_before_refundable_credits,10679.75,
+scenario_099,federal_income_tax_before_refundable_credits,10711.484375,
scenario_099,federal_refundable_credits,0.0,
scenario_099,payroll_tax,14767.5,
scenario_099,self_employment_tax,0.0,
@@ -1723,11 +1723,11 @@ scenario_109,child2_early_head_start_eligible,0.0,0.0
scenario_109,child3_early_head_start_eligible,1.0,21977.013671875
scenario_109,free_school_meals_eligible,1.0,2261.924560546875
scenario_109,reduced_price_school_meals_eligible,0.0,0.0
-scenario_110,federal_income_tax_before_refundable_credits,23897.443359375,
+scenario_110,federal_income_tax_before_refundable_credits,25747.7734375,
scenario_110,federal_refundable_credits,0.0,
scenario_110,payroll_tax,7650.0,
scenario_110,self_employment_tax,0.0,
-scenario_110,state_income_tax_before_refundable_credits,4057.47216796875,
+scenario_110,state_income_tax_before_refundable_credits,4236.8076171875,
scenario_110,state_refundable_credits,0.0,
scenario_110,local_income_tax,0.0,
scenario_110,snap,0.0,
@@ -1913,7 +1913,7 @@ scenario_120,federal_income_tax_before_refundable_credits,40021.81640625,
scenario_120,federal_refundable_credits,0.0,
scenario_120,payroll_tax,13496.1552734375,
scenario_120,self_employment_tax,0.0,
-scenario_120,state_income_tax_before_refundable_credits,11917.18359375,
+scenario_120,state_income_tax_before_refundable_credits,11919.0576171875,
scenario_120,state_refundable_credits,0.0,
scenario_120,local_income_tax,0.0,
scenario_120,snap,0.0,
diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json
index 49293df7..9a563b23 100644
--- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json
+++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json
@@ -11,7 +11,7 @@
"policyengine_version": "6.1.2",
"bundled_policyengine_version": "6.1.2",
"model_package": "policyengine-us",
- "model_version": "2.15.17",
+ "model_version": "2.38.6",
"model_direct_url": null,
"bundled_model_version": "2.2.1",
"model_version_source": "installed package",
@@ -60,7 +60,7 @@
"split": "public",
"split_seed": 1042,
"task": "reference_outputs",
- "reference_csv_sha256": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466",
+ "reference_csv_sha256": "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371",
"revisions": [
{
"date": "2026-09-22",
@@ -1155,7 +1155,1123 @@
"basis": "Moves by $1 or less on policyengine-us 2.15.17; no score changes at the exact-match tolerance."
}
]
+ },
+ {
+ "date": "2026-10-10",
+ "kind": "engine_upgrade",
+ "root_cause": "engine_upgrade_policyengine_us_2_38_6",
+ "outputs": "every scored output",
+ "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. References come from the newest policyengine-us release when PolicyBench begins the reference sweep; at publication PolicyBench checks that the newest release gives the same values. Max, 2026-10-09, on waiting for the release that fixes the engine defects behind PolicyBench's exclusions: 'yes i want to wait for hte fixed engine'. An excluded output whose engine defect the new release fixes, landing on its audited corrected value, is regenerated and its exclusion record removed; every other excluded output keeps the value it was decided on.",
+ "engine_version": "policyengine-us 2.38.6",
+ "previous_engine_version": "policyengine-us 2.15.17",
+ "policyengine_py": "policyengine 6.1.2, provenance only: its certified US bundle is policyengine-us 2.2.1",
+ "fix_modules": [
+ {
+ "module": "latest_c_ca_hold_2025.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py",
+ "sha256": "f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2"
+ },
+ {
+ "module": "latest_c_irs_sales_tax_2025.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py",
+ "sha256": "7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e"
+ },
+ {
+ "module": "latest_c_wi_published_2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py",
+ "sha256": "4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce"
+ },
+ {
+ "module": "latest_c_id_hold_2025.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py",
+ "sha256": "39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5"
+ },
+ {
+ "module": "latest_c_mn_published_2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py",
+ "sha256": "02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522"
+ },
+ {
+ "module": "latest_c_md_2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_md_2026.py",
+ "sha256": "8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76"
+ },
+ {
+ "module": "latest_c_mi_published_2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py",
+ "sha256": "787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e"
+ },
+ {
+ "module": "latest_c_mo_published_2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py",
+ "sha256": "0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916"
+ },
+ {
+ "module": "latest_c_snap_hold_fy2026.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py",
+ "sha256": "3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd"
+ },
+ {
+ "module": "latest_conventions.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_conventions.py",
+ "sha256": "4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb"
+ },
+ {
+ "module": "latest_md_local_output_scope.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py",
+ "sha256": "53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd"
+ },
+ {
+ "module": "latest_final.py",
+ "path": "reference_audit/2026-09-28/fixes/latest_final.py",
+ "sha256": "dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e"
+ },
+ {
+ "module": "r19_irs_sales_tax_2025.json",
+ "path": "reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json",
+ "sha256": "7cf93799aad6a8c3a6fc70e7df4acc6043202a1ecc083f2c3101f76b87a85eae"
+ }
+ ],
+ "builder": "reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py; households from policybench.scenarios.Scenario.to_pe_household through reference_audit/2026-09-28/scripts/sweep.py build_situation, as in the 2026-09-29 upgrade",
+ "provenance": {
+ "builder_sha256": "9fd003be76a7a00ae83e5e17bc652dc7cdee3a7c351fb3efb26308d93c0ca6f5",
+ "harness": "reference_audit/2026-09-28/scripts/sweep.py",
+ "harness_sha256": "f4c7783453a473a1bedcf64ef554c3a2fc29f9d663626948d920ed3ef2ea2b69",
+ "actions_sha256": "ef23b50ec389598a231850d0b87c3635c1306bac965e1ee08cf402d7aad9e201",
+ "release_spec_sha256": "5e7856a42548746bc37760adea1af718050429e0b9c87c4d62755c7d036dab71",
+ "base_commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "policyengine_core": "3.32.29"
+ },
+ "excluded_outputs_untouched": true,
+ "kept_exclusions_from_release": [
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "federal_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "federal_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "state_income_tax_before_refundable_credits"
+ }
+ ],
+ "excluded_outputs_rechecked": [
+ {
+ "scenario_id": "scenario_005",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "kept_value": 106505.8984375,
+ "value_on_2_38_6": 108525.578125,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $108,525.58, on the record's corrected value $108,525.58, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input).); the record keeps $106,505.90."
+ },
+ {
+ "scenario_id": "scenario_007",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 755.7783203125,
+ "value_on_2_38_6": 771.5570068359375,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $771.56, $110.24 from the record's corrected value $661.32, so the defect (r07_idaho_health_premiums) is not fixed in this release; the record keeps $755.78."
+ },
+ {
+ "scenario_id": "scenario_020",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "kept_value": 68056.7109375,
+ "value_on_2_38_6": 68334.734375,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $68,056.71, and policyengine-us 2.38.6 gives $68,334.73. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_023",
+ "variable": "snap",
+ "kept_value": 461.3397216796875,
+ "value_on_2_38_6": 408.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $461.34, and policyengine-us 2.38.6 gives $408.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_042",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 284.74090576171875,
+ "value_on_2_38_6": 464.18280029296875,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $464.18, $464.18 from the record's corrected value $0.00, so the defect (r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions) is not fixed in this release; the record keeps $284.74. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_052",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "kept_value": 104211.40625,
+ "value_on_2_38_6": 104225.2578125,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $104,225.26, on the record's corrected value $104,225.26, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input).); the record keeps $104,211.41."
+ },
+ {
+ "scenario_id": "scenario_053",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 2435.2783203125,
+ "value_on_2_38_6": 2451.05712890625,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $2,451.06, $265.00 from the record's corrected value $2,186.06, so the defect (r07_idaho_health_premiums) is not fixed in this release; the record keeps $2,435.28."
+ },
+ {
+ "scenario_id": "scenario_056",
+ "variable": "snap",
+ "kept_value": 1140.0,
+ "value_on_2_38_6": 95.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $1,140.00, and policyengine-us 2.38.6 gives $95.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_057",
+ "variable": "snap",
+ "kept_value": 2669.217041015625,
+ "value_on_2_38_6": 2628.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $2,669.22, and policyengine-us 2.38.6 gives $2,628.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_080",
+ "variable": "snap",
+ "kept_value": 3596.039794921875,
+ "value_on_2_38_6": 3576.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $3,576.00, $336.00 from the record's corrected value $3,240.00, so the defect (r30_snap_heat_and_eat_sua) is not fixed in this release; the record keeps $3,596.04. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_081",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 8238.40625,
+ "value_on_2_38_6": 8232.900390625,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $8,232.90, on the record's corrected value $8,232.90, but the record names a second reason that a new engine does not settle (Corrected value applies ma_part_a_loss_offset with every publication convention and upstream fix. The output also moves under r23_ma_interest_source (unlisted input).); the record keeps $8,238.41."
+ },
+ {
+ "scenario_id": "scenario_091",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 843.6614990234375,
+ "value_on_2_38_6": 895.8131103515625,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $895.81, $17.30 from the record's corrected value $878.52, so the defect (r32_wi_capital_gain_distributions) is not fixed in this release; the record keeps $843.66. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "state_income_tax_before_refundable_credits",
+ "kept_value": 4493.7431640625,
+ "value_on_2_38_6": 4588.48388671875,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $4,588.48, on the record's audited corrected value, but the reference rests on a further engine defect the new engine does not fix. California does not conform to the federal above-the-line deduction for educator expenses (R&TC 17072; FTB 2025 Instructions for Schedule CA (540), Section C, line 11: 'California law does not conform to federal law regarding educator expenses'), so California AGI adds back the spouse's $337.50 educator expense; policyengine-us sets ca_agi to federal AGI ($159,729.30) without it, and the tax is about $26.90 lower than the law gives. Found by the release's Opus 5.5 judge (reference flag on the re-opened case, 2026-10-10) and confirmed against the FTB instructions; not yet fixed upstream."
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "snap",
+ "kept_value": 8625.8896484375,
+ "value_on_2_38_6": 8556.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $8,625.89, and policyengine-us 2.38.6 gives $8,556.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_112",
+ "variable": "snap",
+ "kept_value": 287.68316650390625,
+ "value_on_2_38_6": 0.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on weekly_hours_worked_before_lsr (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $287.68, and policyengine-us 2.38.6 gives $0.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_118",
+ "variable": "snap",
+ "kept_value": 2903.9404296875,
+ "value_on_2_38_6": 2868.0,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on whether the listed home mortgage interest is on the home the SNAP household occupies (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $2,903.94, and policyengine-us 2.38.6 gives $2,868.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "kept_value": 40021.81640625,
+ "value_on_2_38_6": 40423.17578125,
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $40,423.18, on the record's corrected value $40,423.18, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input). The output also moves under the state income tax withheld reading (unlisted input): the reference's SALT uses the Connecticut withholding estimate ($12,010.05); with the engine's own liability ($11,917.18) the 2.15.17 value is $40,438.95, and with no state income tax paid it is $43,619.93. With r02's IRA fix on 2.15.17 the value is $40,423.18 under the withholding estimate, $40,445.46 under the liability and $43,629.21 with no state income tax paid. policyengine-us 2.15.17 also leaves the household's Connecticut paid leave contributions ($827.98) out of SALT (Rev. Rul. 2025-4); with them the 2.15.17 value is $40,217.95.); the record keeps $40,021.82."
+ }
+ ],
+ "regenerated_exclusions": [
+ {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r01_ira_compensation",
+ "decided_on": "2026-09-22",
+ "kept_value": 22154.69921875,
+ "value_on_2_38_6": 22400.654296875,
+ "audited_alternative_value": 22400.654297,
+ "target": {
+ "kind": "record",
+ "value": 22400.654297,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 22400.654296875,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $22,400.65, within $1 of the corrected value $22,400.65 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "record": {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r01_ira_compensation",
+ "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
+ "frozen_value": 22154.69921875,
+ "alternative_value": 22400.654297,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
+ "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g+r11_ca_itemized_conformity",
+ "decided_on": "2026-09-22",
+ "kept_value": 41051.51171875,
+ "value_on_2_38_6": 40975.25390625,
+ "audited_alternative_value": 40920.402344,
+ "target": {
+ "kind": "fix_modules",
+ "value": 40975.25390625,
+ "engine": "policyengine-us 2.37.2",
+ "engine_value": 41267.01171875,
+ "modules": [
+ {
+ "module": "r02_ira_219g_v2.py",
+ "sha256": "937514fb1199c4ca172dfc5252db7f2483adade193922e6e5283d0b6ab4c4c8c"
+ },
+ {
+ "module": "r11_ca_itemized_conformity_v2.py",
+ "sha256": "2bcb8ba6694c053f3c9a8509b96bfee4976ba00e3fb676c0c504e06c2d6c046d"
+ }
+ ],
+ "dependencies": [
+ {
+ "module": "r02_ira_219g.py",
+ "sha256": "295a0b975ced8964fc1b2c5d52c47bce177eed399f49735eafb88441e7561b84"
+ }
+ ],
+ "evidence": "reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json",
+ "evidence_sha256": "4ccc7160b0d8513dedab1fb60ef57bfbd08cb495f445e7ca955141cb6c6c17a6",
+ "value_with_modules": 40975.25390625
+ },
+ "regenerated": 40975.25390625,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $40,975.25, within $1 of the corrected value $40,975.25 its fix modules (r02_ira_219g_v2.py + r11_ca_itemized_conformity_v2.py) give on policyengine-us 2.37.2, which still has the defect ($41,267.01 without them) (r02_ira_219g+r11_ca_itemized_conformity: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions).",
+ "record": {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g+r11_ca_itemized_conformity",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.",
+ "frozen_value": 41051.51171875,
+ "alternative_value": 40920.402344,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r02_ira_219g_v2 and r11_ca_itemized_conformity_v2 together with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "root_cause": "r05_nj_worker_ui",
+ "decided_on": "2026-09-22",
+ "kept_value": 2162.760009765625,
+ "value_on_2_38_6": 2276.659912109375,
+ "audited_alternative_value": 2276.659912,
+ "target": {
+ "kind": "record",
+ "value": 2276.659912,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 2276.659912109375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10031",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10031) is regenerated with the fix: policyengine-us 2.38.6 gives $2,276.66, within $1 of the corrected value $2,276.66 the 2026-09-22 record computed (r05_nj_worker_ui: PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax).",
+ "record": {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r05_nj_worker_ui",
+ "alternative_reading": "New Jersey workers contribute 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds, up to the 2026 wage base of $44,800, alongside the TDI and FLI contributions.",
+ "frozen_value": 2162.760009765625,
+ "alternative_value": 2276.659912,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax",
+ "law": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r05_nj_worker_ui_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "az_standard_deduction_indexing",
+ "decided_on": "2026-10-06",
+ "kept_value": 1146.05224609375,
+ "value_on_2_38_6": 1137.30224609375,
+ "audited_alternative_value": 1137.302246,
+ "target": {
+ "kind": "record",
+ "value": 1137.302246,
+ "engine": "policyengine-us 2.15.17"
+ },
+ "regenerated": 1137.30224609375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9928",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9928) is regenerated with the fix: policyengine-us 2.38.6 gives $1,137.30, within $1 of the corrected value $1,137.30 the 2026-10-06 record computed (az_standard_deduction_indexing: policyengine-us puts the Arizona standard deduction's uprating on the parent parameter node, which policyengine-core does not propagate to the filing-status children, so the 2025 amount ($15,750 single) carries into 2026 unindexed).",
+ "record": {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen_value": 1146.05224609375,
+ "alternative_value": 1137.302246,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "A.R.S. 43-1041(A)(1) and (H), as amended by Laws 2026, Ch. 140 (HB 4168, approved 2026-06-13); 26 U.S.C. 63(c)(7)(B); Rev. Proc. 2025-32 s. 4.14",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_018__state_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/az_018.md). Alternative value: engine counterfactual: policyengine-us 2.15.17 with latest_final and az_standard_deduction = 16,100 (verification/probes/az_standard_deduction_scenario_018.json).",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "az_standard_deduction_indexing",
+ "alternative_reading": "HB 4168 set the single standard deduction at $15,750 'subject to subsection H', and subsection H requires the department to adjust it 'in the same manner in which the federal basic standard deduction is adjusted for inflation pursuant to section 63'. The federal basic standard deduction starts from the same $15,750 and is $16,100 for 2026 (Rev. Proc. 2025-32), so Arizona's 2026 single deduction is $16,100.",
+ "defect": "policyengine-us puts the Arizona standard deduction's uprating on the parent parameter node, which policyengine-core does not propagate to the filing-status children, so the 2025 amount ($15,750 single) carries into 2026 unindexed",
+ "upstream": "PolicyEngine/policyengine-us#9928"
+ }
+ },
+ {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r11_ca_itemized_conformity",
+ "decided_on": "2026-09-22",
+ "kept_value": 2439.650146484375,
+ "value_on_2_38_6": 1968.805419921875,
+ "audited_alternative_value": 1968.80542,
+ "target": {
+ "kind": "record",
+ "value": 1968.80542,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 1968.805419921875,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10034",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $1,968.81, within $1 of the corrected value $1,968.81 the 2026-09-22 record computed (r11_ca_itemized_conformity: PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions).",
+ "record": {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r11_ca_itemized_conformity",
+ "alternative_reading": "California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.",
+ "frozen_value": 2439.650146484375,
+ "alternative_value": 1968.80542,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions",
+ "law": "Cal. R&TC 17024.5 (conformity date), 17076",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r11_ca_itemized_conformity_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "oh_medical_deduction_premiums",
+ "decided_on": "2026-10-06",
+ "kept_value": 1921.57275390625,
+ "value_on_2_38_6": 1916.606201171875,
+ "audited_alternative_value": 1916.604,
+ "target": {
+ "kind": "record",
+ "value": 1916.604,
+ "engine": "policyengine-us 2.15.17"
+ },
+ "regenerated": 1916.606201171875,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925) is regenerated with the fix: policyengine-us 2.38.6 gives $1,916.61, within $1 of the corrected value $1,916.60 the 2026-10-06 record computed (oh_medical_deduction_premiums: policyengine-us counts health insurance premiums toward Ohio's 7.5%-of-federal-AGI medical deduction only when the person is Medicare-eligible AND has no employer-paid plan, where Ohio's worksheet line 3 reads 'Medicare or an employer-paid health care plan', so a non-Medicare filer with employer coverage gets no premium credit).",
+ "record": {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen_value": 1921.57275390625,
+ "alternative_value": 1916.604,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "R.C. 5747.01(A)(10)(b)-(c) (effective 2026-03-05); 26 U.S.C. 213(d)(1)(D); Ohio IT 1040 instructions, unreimbursed medical and health care expense worksheet line 3",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_025__state_income_tax_before_refundable_credits) and verified independently on 2026-10-06 (verification/independent/oh_025.md). Alternative value: hand computation from the engine's Ohio AGI, exemptions, rate schedule and retirement income credit (verification/independent/oh_025.md section 4).",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "oh_medical_deduction_premiums",
+ "alternative_reading": "R.C. 5747.01(A)(10)(b) deducts unreimbursed medical care above 7.5% of federal AGI, and (A)(10)(c) defines medical care by IRC 213(d), which includes insurance premiums. The head's $6,500 of after-tax non-employer premiums plus $800 of other medical expenses ($7,300) exceed 7.5% of federal AGI ($7,119.40) by $180.60.",
+ "defect": "policyengine-us counts health insurance premiums toward Ohio's 7.5%-of-federal-AGI medical deduction only when the person is Medicare-eligible AND has no employer-paid plan, where Ohio's worksheet line 3 reads 'Medicare or an employer-paid health care plan', so a non-Medicare filer with employer coverage gets no premium credit",
+ "upstream": "PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925"
+ }
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r03_estate_income",
+ "decided_on": "2026-09-22",
+ "kept_value": 1345.510498046875,
+ "value_on_2_38_6": 8596.0283203125,
+ "audited_alternative_value": 8596.02832,
+ "target": {
+ "kind": "record",
+ "value": 8596.02832,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 8596.0283203125,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $8,596.03, within $1 of the corrected value $8,596.03 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "record": {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r03_estate_income",
+ "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
+ "frozen_value": 1345.510498046875,
+ "alternative_value": 8596.02832,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
+ "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
+ "upstream": "related: PolicyEngine/policyengine-us#9304",
+ "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r03_estate_income",
+ "decided_on": "2026-09-22",
+ "kept_value": 514.4984130859375,
+ "value_on_2_38_6": 1975.7982177734375,
+ "audited_alternative_value": 1975.798218,
+ "target": {
+ "kind": "record",
+ "value": 1975.798218,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 1975.7982177734375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9633",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9633) is regenerated with the fix: policyengine-us 2.38.6 gives $1,975.80, within $1 of the corrected value $1,975.80 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "record": {
+ "scenario_id": "scenario_039",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r03_estate_income",
+ "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
+ "frozen_value": 514.4984130859375,
+ "alternative_value": 1975.798218,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
+ "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
+ "upstream": "related: PolicyEngine/policyengine-us#9304",
+ "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "root_cause": "co_sales_tax_refund_surplus",
+ "decided_on": "2026-10-06",
+ "kept_value": 19.0,
+ "value_on_2_38_6": 0.0,
+ "audited_alternative_value": 0.0,
+ "target": {
+ "kind": "record",
+ "value": 0.0,
+ "engine": "policyengine-us 2.15.17"
+ },
+ "regenerated": 0.0,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9946",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9946) is regenerated with the fix: policyengine-us 2.38.6 gives $0.00, within $1 of the corrected value $0.00 the 2026-10-06 record computed (co_sales_tax_refund_surplus: policyengine-us pays Colorado's six-tier state sales tax refund without the statute's condition that the fiscal year ending in the tax year had excess state revenues, carrying the tax year 2025 table into 2026).",
+ "record": {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "frozen_value": 19.0,
+ "alternative_value": 0.0,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "C.R.S. 39-22-2003(2), 39-22-2002(2)(a), 39-3-209; Colorado Legislative Council Staff forecasts of December 2025, March 2026 and June 2026 (tax year 2026 refund table $0 in every tier); OSPB forecast of 2026-06-18",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_043__state_refundable_credits) and verified independently on 2026-10-06 (verification/independent/co_043.md). Alternative value: law: no tax year 2026 sales tax refund; the household has no other Colorado refundable credit (verification/independent/co_043.md section 5).",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "co_sales_tax_refund_surplus",
+ "alternative_reading": "C.R.S. 39-22-2003(2) allows the refund for a tax year only 'if there were excess state revenues for the fiscal year ending in that tax year'. Each Legislative Council Staff forecast before the freeze (December 2025, March and June 2026) published a tax year 2026 refund table of $0 in every tier, OSPB's June 2026 outlook put FY 2025-26 $15.3 million below the cap and anticipated no refunds, and no pre-freeze official source described a tax year 2026 six-tier refund; the State Controller certified a $175.9 million shortfall on 2026-09-08.",
+ "defect": "policyengine-us pays Colorado's six-tier state sales tax refund without the statute's condition that the fiscal year ending in the tax year had excess state revenues, carrying the tax year 2025 table into 2026",
+ "upstream": "PolicyEngine/policyengine-us#9946"
+ }
+ },
+ {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g",
+ "decided_on": "2026-09-22",
+ "kept_value": 30543.908203125,
+ "value_on_2_38_6": 30702.58984375,
+ "audited_alternative_value": 30702.589844,
+ "target": {
+ "kind": "record",
+ "value": 30702.589844,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 30702.58984375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $30,702.59, within $1 of the corrected value $30,702.59 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "record": {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
+ "frozen_value": 30543.908203125,
+ "alternative_value": 30702.589844,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r01_ira_compensation",
+ "decided_on": "2026-09-22",
+ "kept_value": 4439.28955078125,
+ "value_on_2_38_6": 4441.455078125,
+ "audited_alternative_value": 4441.455078,
+ "target": {
+ "kind": "record",
+ "value": 4441.455078,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 4441.455078125,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9801",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,441.46, within $1 of the corrected value $4,441.46 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "record": {
+ "scenario_id": "scenario_064",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r01_ira_compensation",
+ "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
+ "frozen_value": 4439.28955078125,
+ "alternative_value": 4441.455078,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
+ "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r01_ira_compensation",
+ "decided_on": "2026-09-22",
+ "kept_value": 4605.9970703125,
+ "value_on_2_38_6": 4599.62158203125,
+ "audited_alternative_value": 4599.621582,
+ "target": {
+ "kind": "record",
+ "value": 4599.621582,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 4599.62158203125,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9801",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,599.62, within $1 of the corrected value $4,599.62 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "record": {
+ "scenario_id": "scenario_064",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r01_ira_compensation",
+ "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.",
+ "frozen_value": 4605.9970703125,
+ "alternative_value": 4599.621582,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return",
+ "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g",
+ "decided_on": "2026-09-22",
+ "kept_value": 9563.052734375,
+ "value_on_2_38_6": 9579.767578125,
+ "audited_alternative_value": 9564.916992,
+ "target": {
+ "kind": "fix_modules",
+ "value": 9579.767578125,
+ "engine": "policyengine-us 2.37.2",
+ "engine_value": 9577.90234375,
+ "modules": [
+ {
+ "module": "r02_ira_219g_v2.py",
+ "sha256": "937514fb1199c4ca172dfc5252db7f2483adade193922e6e5283d0b6ab4c4c8c"
+ }
+ ],
+ "dependencies": [
+ {
+ "module": "r02_ira_219g.py",
+ "sha256": "295a0b975ced8964fc1b2c5d52c47bce177eed399f49735eafb88441e7561b84"
+ }
+ ],
+ "evidence": "reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json",
+ "evidence_sha256": "4ccc7160b0d8513dedab1fb60ef57bfbd08cb495f445e7ca955141cb6c6c17a6",
+ "value_with_modules": 9579.767578125
+ },
+ "regenerated": 9579.767578125,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $9,579.77, within $1 of the corrected value $9,579.77 its fix modules (r02_ira_219g_v2.py) give on policyengine-us 2.37.2, which still has the defect ($9,577.90 without them) (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "record": {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
+ "frozen_value": 9563.052734375,
+ "alternative_value": 9564.916992,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "root_cause": "ny_cdcc_606_c2",
+ "decided_on": "2026-10-06",
+ "kept_value": 667.0,
+ "value_on_2_38_6": 1187.5478515625,
+ "audited_alternative_value": 1187.611,
+ "target": {
+ "kind": "record",
+ "value": 1187.611,
+ "engine": "policyengine-us 2.15.17"
+ },
+ "regenerated": 1187.5478515625,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9948",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9948) is regenerated with the fix: policyengine-us 2.38.6 gives $1,187.55, within $1 of the corrected value $1,187.61 the 2026-10-06 record computed (ny_cdcc_606_c2: policyengine-us computes New York's 2026 child and dependent care credit with the pre-2026 formula (a percentage of the federal credit), which L.2026 ch. 59 limited to tax years beginning before 2026 and replaced with the refundable 606(c-2) credit on qualifying expenses).",
+ "record": {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "frozen_value": 667.0,
+ "alternative_value": 1187.611,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d1022",
+ "law": "N.Y. Tax Law 606(c)(1) and 606(c-2), as amended and added by L.2026, ch. 59, Part A (S.9009-C/A.10009-C, signed 2026-05-28)",
+ "note": "Found 2026-10-05 by the reference adversary (runs/claude/cases/us__scenario_082__state_refundable_credits) and verified independently on 2026-10-06 (verification/independent/ny_082.md). Alternative value: hand computation from the engine's New York AGI and Empire State child credit (verification/independent/ny_082.md section 3).",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "ny_cdcc_606_c2",
+ "alternative_reading": "Tax Law 606(c-2) gives, for tax years beginning on or after 2026-01-01, a refundable credit of the applicable percentage (55%, less 0.00025 percentage points per dollar of New York AGI above $15,000, floor 4%) of qualifying expenses up to $3,000 for one child. At New York AGI of $117,585.15 that is 29.3537% of $3,000 = $880.61, plus the $307 Empire State child credit.",
+ "defect": "policyengine-us computes New York's 2026 child and dependent care credit with the pre-2026 formula (a percentage of the federal credit), which L.2026 ch. 59 limited to tax years beginning before 2026 and replaced with the refundable 606(c-2) credit on qualifying expenses",
+ "upstream": "PolicyEngine/policyengine-us#9948"
+ }
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g",
+ "decided_on": "2026-09-22",
+ "kept_value": 10679.75,
+ "value_on_2_38_6": 10711.484375,
+ "audited_alternative_value": 10711.484375,
+ "target": {
+ "kind": "record",
+ "value": 10711.484375,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 10711.484375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $10,711.48, within $1 of the corrected value $10,711.48 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "record": {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
+ "frozen_value": 10679.75,
+ "alternative_value": 10711.484375,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g+r03_estate_income",
+ "decided_on": "2026-09-22",
+ "kept_value": 23897.443359375,
+ "value_on_2_38_6": 25747.7734375,
+ "audited_alternative_value": 25747.773438,
+ "target": {
+ "kind": "record",
+ "value": 25747.773438,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 25747.7734375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $25,747.77, within $1 of the corrected value $25,747.77 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "record": {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g+r03_estate_income",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
+ "frozen_value": 23897.443359375,
+ "alternative_value": 25747.773438,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
+ "upstream": "related: PolicyEngine/policyengine-us#9304",
+ "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g+r03_estate_income",
+ "decided_on": "2026-09-22",
+ "kept_value": 4057.47216796875,
+ "value_on_2_38_6": 4236.8076171875,
+ "audited_alternative_value": 4236.807617,
+ "target": {
+ "kind": "record",
+ "value": 4236.807617,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 4236.8076171875,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $4,236.81, within $1 of the corrected value $4,236.81 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "record": {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g+r03_estate_income",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.",
+ "frozen_value": 4057.47216796875,
+ "alternative_value": 4236.807617,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)",
+ "upstream": "related: PolicyEngine/policyengine-us#9304",
+ "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix."
+ }
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "root_cause": "r02_ira_219g",
+ "decided_on": "2026-09-22",
+ "kept_value": 11917.18359375,
+ "value_on_2_38_6": 11919.0576171875,
+ "audited_alternative_value": 11919.057617,
+ "target": {
+ "kind": "record",
+ "value": 11919.057617,
+ "engine": "policyengine-us 1.755.4"
+ },
+ "regenerated": 11919.0576171875,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $11,919.06, within $1 of the corrected value $11,919.06 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "record": {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_engine_defect",
+ "root_cause": "r02_ira_219g",
+ "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.",
+ "frozen_value": 11917.18359375,
+ "alternative_value": 11919.057617,
+ "engine_version": "policyengine-us 1.755.4",
+ "decided_on": "2026-09-22",
+ "decided_by": "developer",
+ "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out",
+ "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
+ "upstream": "to be filed",
+ "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix."
+ }
+ }
+ ],
+ "new_exclusions": [
+ {
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "frozen_value": 670.5501098632812,
+ "alternative_value": 0.0,
+ "decided_on": "2026-10-10"
+ },
+ {
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "frozen_value": 1348.83203125,
+ "alternative_value": 0.0,
+ "decided_on": "2026-10-10"
+ }
+ ],
+ "changed": [
+ {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 22154.69921875,
+ "previous": 22154.69921875,
+ "regenerated": 22400.654296875,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $22,400.65, within $1 of the corrected value $22,400.65 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return)."
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 41051.51171875,
+ "previous": 41051.51171875,
+ "regenerated": 40975.25390625,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $40,975.25, within $1 of the corrected value $40,975.25 its fix modules (r02_ira_219g_v2.py + r11_ca_itemized_conformity_v2.py) give on policyengine-us 2.37.2, which still has the defect ($41,267.01 without them) (r02_ira_219g+r11_ca_itemized_conformity: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions)."
+ },
+ {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "frozen": 2162.760009765625,
+ "previous": 2162.760009765625,
+ "regenerated": 2276.659912109375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10031) is regenerated with the fix: policyengine-us 2.38.6 gives $2,276.66, within $1 of the corrected value $2,276.66 the 2026-09-22 record computed (r05_nj_worker_ui: PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax)."
+ },
+ {
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "frozen": 0.0,
+ "previous": 0.0,
+ "regenerated": 670.5501098632812,
+ "cause": "excluded_reference_depends_on_unlisted_input",
+ "basis": "Newly excluded from scoring on 2026-10-10 (reference_exclusions.json): county of residence (Indiana county income tax)"
+ },
+ {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 1146.05224609375,
+ "previous": 1146.05224609375,
+ "regenerated": 1137.30224609375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9928) is regenerated with the fix: policyengine-us 2.38.6 gives $1,137.30, within $1 of the corrected value $1,137.30 the 2026-10-06 record computed (az_standard_deduction_indexing: policyengine-us puts the Arizona standard deduction's uprating on the parent parameter node, which policyengine-core does not propagate to the filing-status children, so the 2025 amount ($15,750 single) carries into 2026 unindexed)."
+ },
+ {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 2439.650146484375,
+ "previous": 2439.650146484375,
+ "regenerated": 1968.805419921875,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $1,968.81, within $1 of the corrected value $1,968.81 the 2026-09-22 record computed (r11_ca_itemized_conformity: PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions)."
+ },
+ {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 1921.57275390625,
+ "previous": 1921.57275390625,
+ "regenerated": 1916.606201171875,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925) is regenerated with the fix: policyengine-us 2.38.6 gives $1,916.61, within $1 of the corrected value $1,916.60 the 2026-10-06 record computed (oh_medical_deduction_premiums: policyengine-us counts health insurance premiums toward Ohio's 7.5%-of-federal-AGI medical deduction only when the person is Medicare-eligible AND has no employer-paid plan, where Ohio's worksheet line 3 reads 'Medicare or an employer-paid health care plan', so a non-Medicare filer with employer coverage gets no premium credit)."
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 1345.510498046875,
+ "previous": 1345.510498046875,
+ "regenerated": 8596.0283203125,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $8,596.03, within $1 of the corrected value $8,596.03 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income)."
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 514.4984130859375,
+ "previous": 514.4984130859375,
+ "regenerated": 1975.7982177734375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9633) is regenerated with the fix: policyengine-us 2.38.6 gives $1,975.80, within $1 of the corrected value $1,975.80 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income)."
+ },
+ {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "frozen": 19.0,
+ "previous": 19.0,
+ "regenerated": 0.0,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9946) is regenerated with the fix: policyengine-us 2.38.6 gives $0.00, within $1 of the corrected value $0.00 the 2026-10-06 record computed (co_sales_tax_refund_surplus: policyengine-us pays Colorado's six-tier state sales tax refund without the statute's condition that the fiscal year ending in the tax year had excess state revenues, carrying the tax year 2025 table into 2026)."
+ },
+ {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 30543.908203125,
+ "previous": 30543.908203125,
+ "regenerated": 30702.58984375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $30,702.59, within $1 of the corrected value $30,702.59 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out)."
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 4439.28955078125,
+ "previous": 4439.28955078125,
+ "regenerated": 4441.455078125,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,441.46, within $1 of the corrected value $4,441.46 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return)."
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 4605.9970703125,
+ "previous": 4605.9970703125,
+ "regenerated": 4599.62158203125,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,599.62, within $1 of the corrected value $4,599.62 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return)."
+ },
+ {
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "frozen": 0.0,
+ "previous": 0.0,
+ "regenerated": 1348.83203125,
+ "cause": "excluded_reference_depends_on_unlisted_input",
+ "basis": "Newly excluded from scoring on 2026-10-10 (reference_exclusions.json): county of residence (Indiana county income tax)"
+ },
+ {
+ "scenario_id": "scenario_076",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 6806.78662109375,
+ "previous": 6818.34423828125,
+ "regenerated": 6828.34423828125,
+ "cause": "id_permanent_building_fund_tax",
+ "basis": "Idaho's $10 permanent building fund tax counts in state income tax: on policyengine-us 2.38.6, state_income_tax_before_refundable_credits adds id_pbf ($10 for a filer liable for it, id_pbf_liable; Idaho Form 40 instructions) to Idaho income tax before refundable credits, since policyengine-us 2.30.0 (PolicyEngine/policyengine-us#9810)."
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 9563.052734375,
+ "previous": 9563.052734375,
+ "regenerated": 9579.767578125,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $9,579.77, within $1 of the corrected value $9,579.77 its fix modules (r02_ira_219g_v2.py) give on policyengine-us 2.37.2, which still has the defect ($9,577.90 without them) (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out)."
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "frozen": 650.5,
+ "previous": 667.0,
+ "regenerated": 1187.5478515625,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9948) is regenerated with the fix: policyengine-us 2.38.6 gives $1,187.55, within $1 of the corrected value $1,187.61 the 2026-10-06 record computed (ny_cdcc_606_c2: policyengine-us computes New York's 2026 child and dependent care credit with the pre-2026 formula (a percentage of the federal credit), which L.2026 ch. 59 limited to tax years beginning before 2026 and replaced with the refundable 606(c-2) credit on qualifying expenses)."
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 10679.75,
+ "previous": 10679.75,
+ "regenerated": 10711.484375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $10,711.48, within $1 of the corrected value $10,711.48 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out)."
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 23897.443359375,
+ "previous": 23897.443359375,
+ "regenerated": 25747.7734375,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $25,747.77, within $1 of the corrected value $25,747.77 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income)."
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 4057.47216796875,
+ "previous": 4057.47216796875,
+ "regenerated": 4236.8076171875,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $4,236.81, within $1 of the corrected value $4,236.81 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income)."
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 11917.18359375,
+ "previous": 11917.18359375,
+ "regenerated": 11919.0576171875,
+ "cause": "regenerated_upstream_fix",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $11,919.06, within $1 of the corrected value $11,919.06 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out)."
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_income_tax_before_refundable_credits",
+ "frozen": 5598.55224609375,
+ "previous": 5598.55224609375,
+ "regenerated": 5599.1435546875,
+ "cause": "engine_upgrade_within_1",
+ "basis": "Moves by $1 or less on policyengine-us 2.38.6; no score changes at the exact-match tolerance."
+ },
+ {
+ "scenario_id": "scenario_085",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "frozen": 2322.760009765625,
+ "previous": 2322.760009765625,
+ "regenerated": 2322.8486328125,
+ "cause": "engine_upgrade_within_1",
+ "basis": "Moves by $1 or less on policyengine-us 2.38.6; no score changes at the exact-match tolerance."
+ }
+ ]
}
],
- "regenerated_at_utc": "2026-09-29T15:04:45.724738+00:00"
+ "regenerated_at_utc": "2026-10-10T06:39:57.084656+00:00"
}
diff --git a/paper/snapshot/20260501/us_impact_summary_by_model.csv b/paper/snapshot/20260501/us_impact_summary_by_model.csv
index d7a013c2..303eed76 100644
--- a/paper/snapshot/20260501/us_impact_summary_by_model.csv
+++ b/paper/snapshot/20260501/us_impact_summary_by_model.csv
@@ -1,47 +1,48 @@
model,mean_impact_score,mean_household_score,mean_household_coverage,households,total_variables,parsed_variables,floor_share
-gpt-6-astra,0.9683122444817193,0.9784738250614189,1.0,100,1920,1920,0.3
-gpt-6-sol,0.9667129921990232,0.9825416199504653,1.0,100,1920,1920,0.3
-gpt-6-luna,0.9664167571502716,0.977394792016256,1.0,100,1920,1920,0.3
-claude-opus-5.5,0.962246515529886,0.982664785613371,1.0,100,1920,1920,0.3
-gpt-5.6-sol,0.9599111614346824,0.9765950579493266,1.0,100,1920,1920,0.3
-gpt-6.1-sol,0.9594811057246667,0.975488445843518,1.0,100,1920,1920,0.3
-claude-sonnet-5.5,0.9593949353595599,0.9733090827566719,1.0,100,1920,1920,0.3
-claude-fable-5.1,0.9569037993263287,0.9780947033412107,1.0,100,1920,1920,0.3
-grok-4.7,0.9544396463560192,0.9761065254261345,1.0,100,1920,1920,0.3
-inkling,0.9537243518627281,0.9708950810157603,1.0,100,1920,1920,0.3
-gpt-5.5,0.9463990239340224,0.9662756610360458,1.0,100,1920,1920,0.3
-deepseek-v4.1-flash,0.9449297542963264,0.9677712793442468,1.0,100,1920,1920,0.3
-gpt-5.6-terra,0.9428227924598422,0.9653030686475066,1.0,100,1920,1920,0.3
-gemini-3.8-flash,0.9413625215070209,0.9757756765309048,1.0,100,1920,1920,0.3
-grok-4.6,0.9370631371759138,0.9709023505552119,1.0,100,1920,1920,0.3
-gemini-3.6-flash,0.9353602488202646,0.9710758545154309,1.0,100,1920,1920,0.3
-ox-alpha,0.9349263306800009,0.9645184800680184,1.0,100,1920,1920,0.3
-kimi-k3,0.9321859170038987,0.947074103380015,0.97,100,1920,1862,0.3
-grok-4.5,0.9265179738707723,0.9637656553313277,1.0,100,1920,1920,0.3
-deepseek-v4-pro-0813,0.92592710415014,0.9666373591127111,1.0,100,1920,1920,0.3
-gpt-5.6-luna,0.9214893430494332,0.9595527347680398,1.0,100,1920,1920,0.3
-gemini-3-flash-preview,0.9194570538721496,0.9616863739278441,1.0,100,1920,1920,0.3
-deepseek-v4-flash-0731,0.9189428784905118,0.9549362454617714,1.0,100,1920,1920,0.3
-claude-opus-4.7,0.9178204428076171,0.9341214404196417,1.0,100,1920,1920,0.3
-claude-sonnet-4.6,0.9162129440743614,0.9452099117398467,1.0,100,1920,1920,0.3
-claude-fable-5,0.9137887992654609,0.9541658659499054,1.0,100,1920,1920,0.3
-claude-opus-5,0.9117042199103389,0.9569245656983695,1.0,100,1920,1920,0.3
-gemini-3.7-flash,0.9080053412243946,0.9640672785413511,1.0,100,1920,1920,0.3
-grok-build-0.1,0.907229168647409,0.9495641986927762,1.0,100,1920,1920,0.3
-gemini-3.1-pro-preview,0.9015422572930285,0.96182457021108,1.0,100,1920,1920,0.3
-deepseek-v4-pro,0.894759815215543,0.9531689498921042,1.0,100,1920,1920,0.3
-gemini-3.5-flash,0.8945050366123243,0.9571534285510584,1.0,100,1920,1920,0.3
-glm-5.3,0.8897481095502663,0.9276676213475469,0.97,100,1920,1849,0.3
-claude-opus-4.8,0.8826928093313342,0.9195533132045565,1.0,100,1920,1920,0.3
-claude-sonnet-5,0.8666788054918853,0.9162508180749187,1.0,100,1920,1920,0.3
-gemini-3.1-flash-lite-preview,0.8635748475506705,0.940345176887288,1.0,100,1920,1920,0.3
-qwen-3.7-max,0.8582890324943854,0.9260164229773913,1.0,100,1920,1920,0.3
-grok-4.3,0.8557494611738747,0.9440812980726867,1.0,100,1920,1920,0.3
-glm-5.2,0.8465447008531164,0.8995325933970066,0.9433301746876838,100,1920,1788,0.3
-qwen3.8-max,0.822354297921981,0.9071986559772072,1.0,100,1920,1920,0.3
-gemini-3.5-flash-lite,0.8157005424440896,0.9412320459816107,1.0,100,1920,1920,0.3
-claude-haiku-4.5,0.7972978664052773,0.9093026578365233,1.0,100,1920,1920,0.3
-gpt-5.4-mini,0.7646496681088202,0.9044299489482875,1.0,100,1920,1920,0.3
-minimax-m3,0.7486300831203856,0.9185494731869339,1.0,100,1920,1920,0.3
-gpt-5.4-nano,0.6976458164890331,0.8943748089883199,1.0,100,1920,1920,0.3
-kimi-k2.6,0.5929836623884213,0.7777325674444523,0.796733767182861,100,1920,1537,0.3
+gpt-6-sol,0.9683890379175923,0.9835781550408013,1.0,100,1926,1926,0.3
+gpt-6-astra,0.9656360677841757,0.9776432719593253,1.0,100,1926,1926,0.3
+gpt-6-luna,0.9655856177136641,0.977552888689478,1.0,100,1926,1926,0.3
+claude-opus-5.5,0.9645893550945023,0.983719384804741,1.0,100,1926,1926,0.3
+gpt-5.6-sol,0.9603411979737442,0.9772470545835048,1.0,100,1926,1926,0.3
+claude-fable-5.1,0.9596842169634039,0.9782975576452126,1.0,100,1926,1926,0.3
+gpt-6.1-sol,0.9577997782276715,0.9755996956654386,1.0,100,1926,1926,0.3
+claude-sonnet-5.5,0.9557169975683331,0.9732048971867467,1.0,100,1926,1926,0.3
+inkling,0.9553730764952827,0.9718310312144887,1.0,100,1926,1926,0.3
+grok-4.7,0.9530593671514012,0.9769017229703395,1.0,100,1926,1926,0.3
+gemini-3.8-flash,0.9462699389421277,0.977767902256248,1.0,100,1926,1926,0.3
+gpt-5.5,0.9453055193873112,0.9667817665830087,1.0,100,1926,1926,0.3
+deepseek-v4.1-flash,0.9436620453469386,0.9685038058811106,1.0,100,1926,1926,0.3
+gpt-5.6-terra,0.9412373765543233,0.9658277780925907,1.0,100,1926,1926,0.3
+ox-alpha,0.9384675548301299,0.9658066001187361,1.0,100,1926,1926,0.3
+kimi-k3,0.9369093783380158,0.9485146940042462,0.97,100,1926,1866,0.3
+gemini-3.6-flash,0.9351220326362744,0.9716546088284257,1.0,100,1926,1926,0.3
+grok-4.6,0.9338430747098238,0.9716311365558624,1.0,100,1926,1926,0.3
+grok-4.5,0.9247602994174631,0.9648720629656492,1.0,100,1926,1926,0.3
+deepseek-v4-pro-0813,0.9212125402654001,0.9671921672284504,1.0,100,1926,1926,0.3
+gpt-5.6-luna,0.9203959222079253,0.9600106363757288,1.0,100,1926,1926,0.3
+deepseek-v4-flash-0731,0.9173624769435257,0.9550956395701116,1.0,100,1926,1926,0.3
+gemini-3-flash-preview,0.9161481688270676,0.9621720429043401,1.0,100,1926,1926,0.3
+claude-opus-4.7,0.9117276162674928,0.9329098370223292,1.0,100,1926,1926,0.3
+claude-sonnet-4.6,0.9104613797385994,0.9449727716388616,1.0,100,1926,1926,0.3
+gemini-3.7-flash,0.907486043649933,0.9647129992653505,1.0,100,1926,1926,0.3
+grok-build-0.1,0.905886694331183,0.9499599365076756,1.0,100,1926,1926,0.3
+claude-fable-5,0.9054051934217692,0.9537747319332733,1.0,100,1926,1926,0.3
+claude-opus-5,0.9052680685413241,0.956209808032048,1.0,100,1926,1926,0.3
+gemini-3.1-pro-preview,0.9002961530624367,0.9625050398352599,1.0,100,1926,1926,0.3
+gemini-3.5-flash,0.8943613983201546,0.9582086228255206,1.0,100,1926,1926,0.3
+deepseek-v4-pro,0.8891412957260344,0.9530685519112764,1.0,100,1926,1926,0.3
+glm-5.3,0.8863036650145125,0.9270981657064452,0.97,100,1926,1853,0.3
+claude-opus-4.8,0.8763670208984979,0.9193961589624415,1.0,100,1926,1926,0.3
+gemini-3.1-flash-lite-preview,0.8549265116548262,0.9400571073735258,1.0,100,1926,1926,0.3
+claude-sonnet-5,0.8546597554533697,0.9145775517501825,1.0,100,1926,1926,0.3
+grok-4.3,0.8444550761064535,0.9431852757043429,1.0,100,1926,1926,0.3
+glm-5.2,0.8428927526586356,0.898248786758015,0.942134973522516,100,1926,1790,0.3
+qwen-3.7-max,0.8413035116697489,0.9245943379074073,1.0,100,1926,1926,0.3
+qwen3.8-max,0.8145343660950313,0.9054694717193006,1.0,100,1926,1926,0.3
+gemini-3.5-flash-lite,0.8048661218683532,0.9403942740096725,1.0,100,1926,1926,0.3
+claude-haiku-4.5,0.7820055915532537,0.9079458048084028,1.0,100,1926,1926,0.3
+claude-haiku-5.5,0.7508491615855171,0.9318432564796563,1.0,100,1926,1926,0.3
+gpt-5.4-mini,0.7438501974068404,0.901965060179966,1.0,100,1926,1926,0.3
+minimax-m3,0.7363778997274973,0.9163087437899782,1.0,100,1926,1926,0.3
+gpt-5.4-nano,0.6816490778744461,0.8929229972630024,1.0,100,1926,1926,0.3
+kimi-k2.6,0.5822018394629117,0.7752609667992933,0.7938565674923747,100,1926,1536,0.3
diff --git a/paper/snapshot/20260501/us_reference_outputs.csv b/paper/snapshot/20260501/us_reference_outputs.csv
index 67d4e5ae..323a4522 100644
--- a/paper/snapshot/20260501/us_reference_outputs.csv
+++ b/paper/snapshot/20260501/us_reference_outputs.csv
@@ -55,7 +55,7 @@ scenario_002,head_medicare_eligible,1.0,5285.2001953125
scenario_002,spouse_medicare_eligible,1.0,5285.2001953125
scenario_002,free_school_meals_eligible,0.0,0.0
scenario_002,reduced_price_school_meals_eligible,0.0,0.0
-scenario_003,federal_income_tax_before_refundable_credits,22154.69921875,
+scenario_003,federal_income_tax_before_refundable_credits,22400.654296875,
scenario_003,federal_refundable_credits,0.0,
scenario_003,payroll_tax,0.0,
scenario_003,self_employment_tax,0.0,
@@ -99,7 +99,7 @@ scenario_005,federal_income_tax_before_refundable_credits,106505.8984375,
scenario_005,federal_refundable_credits,0.0,
scenario_005,payroll_tax,36044.0,
scenario_005,self_employment_tax,0.0,
-scenario_005,state_income_tax_before_refundable_credits,41051.51171875,
+scenario_005,state_income_tax_before_refundable_credits,40975.25390625,
scenario_005,state_refundable_credits,0.0,
scenario_005,local_income_tax,0.0,
scenario_005,snap,0.0,
@@ -133,7 +133,7 @@ scenario_007,free_school_meals_eligible,0.0,0.0
scenario_007,reduced_price_school_meals_eligible,0.0,0.0
scenario_008,federal_income_tax_before_refundable_credits,0.0,
scenario_008,federal_refundable_credits,12433.611328125,
-scenario_008,payroll_tax,2162.760009765625,
+scenario_008,payroll_tax,2276.659912109375,
scenario_008,self_employment_tax,565.1820068359375,
scenario_008,state_income_tax_before_refundable_credits,264.80999755859375,
scenario_008,state_refundable_credits,5842.39990234375,
@@ -275,7 +275,7 @@ scenario_015,payroll_tax,3442.5,
scenario_015,self_employment_tax,0.0,
scenario_015,state_income_tax_before_refundable_credits,1236.3267822265625,
scenario_015,state_refundable_credits,0.0,
-scenario_015,local_income_tax,0.0,
+scenario_015,local_income_tax,670.5501098632812,
scenario_015,snap,0.0,
scenario_015,ssi,0.0,
scenario_015,tanf,0.0,
@@ -305,7 +305,7 @@ scenario_018,federal_income_tax_before_refundable_credits,5211.05078125,
scenario_018,federal_refundable_credits,0.0,
scenario_018,payroll_tax,5202.38232421875,
scenario_018,self_employment_tax,0.0,
-scenario_018,state_income_tax_before_refundable_credits,1146.05224609375,
+scenario_018,state_income_tax_before_refundable_credits,1137.30224609375,
scenario_018,state_refundable_credits,0.0,
scenario_018,local_income_tax,0.0,
scenario_018,snap,0.0,
@@ -353,7 +353,7 @@ scenario_022,federal_income_tax_before_refundable_credits,11113.5703125,
scenario_022,federal_refundable_credits,0.0,
scenario_022,payroll_tax,4710.83251953125,
scenario_022,self_employment_tax,0.0,
-scenario_022,state_income_tax_before_refundable_credits,2439.650146484375,
+scenario_022,state_income_tax_before_refundable_credits,1968.805419921875,
scenario_022,state_refundable_credits,0.0,
scenario_022,local_income_tax,0.0,
scenario_022,snap,0.0,
@@ -385,7 +385,7 @@ scenario_025,federal_income_tax_before_refundable_credits,7031.03564453125,
scenario_025,federal_refundable_credits,0.0,
scenario_025,payroll_tax,4798.48486328125,
scenario_025,self_employment_tax,0.0,
-scenario_025,state_income_tax_before_refundable_credits,1921.57275390625,
+scenario_025,state_income_tax_before_refundable_credits,1916.606201171875,
scenario_025,state_refundable_credits,0.0,
scenario_025,local_income_tax,0.0,
scenario_025,snap,0.0,
@@ -651,11 +651,11 @@ scenario_038,child1_early_head_start_eligible,0.0,0.0
scenario_038,child2_early_head_start_eligible,0.0,0.0
scenario_038,free_school_meals_eligible,1.0,2261.924560546875
scenario_038,reduced_price_school_meals_eligible,0.0,0.0
-scenario_039,federal_income_tax_before_refundable_credits,1345.510498046875,
+scenario_039,federal_income_tax_before_refundable_credits,8596.0283203125,
scenario_039,federal_refundable_credits,0.0,
scenario_039,payroll_tax,0.0,
scenario_039,self_employment_tax,0.0,
-scenario_039,state_income_tax_before_refundable_credits,514.4984130859375,
+scenario_039,state_income_tax_before_refundable_credits,1975.7982177734375,
scenario_039,state_refundable_credits,0.0,
scenario_039,local_income_tax,0.0,
scenario_039,snap,0.0,
@@ -708,7 +708,7 @@ scenario_043,federal_refundable_credits,0.0,
scenario_043,payroll_tax,330.7310791015625,
scenario_043,self_employment_tax,0.0,
scenario_043,state_income_tax_before_refundable_credits,0.0,
-scenario_043,state_refundable_credits,19.0,
+scenario_043,state_refundable_credits,0.0,
scenario_043,local_income_tax,0.0,
scenario_043,snap,3576.0,
scenario_043,ssi,0.0,
@@ -803,7 +803,7 @@ scenario_048,head_chip_eligible,0.0,0.0
scenario_048,head_medicare_eligible,1.0,5285.2001953125
scenario_048,free_school_meals_eligible,0.0,0.0
scenario_048,reduced_price_school_meals_eligible,0.0,0.0
-scenario_049,federal_income_tax_before_refundable_credits,30543.908203125,
+scenario_049,federal_income_tax_before_refundable_credits,30702.58984375,
scenario_049,federal_refundable_credits,0.0,
scenario_049,payroll_tax,15134.5,
scenario_049,self_employment_tax,0.0,
@@ -1007,11 +1007,11 @@ scenario_062,head_chip_eligible,0.0,0.0
scenario_062,head_medicare_eligible,1.0,5285.2001953125
scenario_062,free_school_meals_eligible,0.0,0.0
scenario_062,reduced_price_school_meals_eligible,0.0,0.0
-scenario_064,federal_income_tax_before_refundable_credits,4439.28955078125,
+scenario_064,federal_income_tax_before_refundable_credits,4441.455078125,
scenario_064,federal_refundable_credits,0.0,
scenario_064,payroll_tax,7443.0673828125,
scenario_064,self_employment_tax,0.0,
-scenario_064,state_income_tax_before_refundable_credits,4605.9970703125,
+scenario_064,state_income_tax_before_refundable_credits,4599.62158203125,
scenario_064,state_refundable_credits,0.0,
scenario_064,local_income_tax,0.0,
scenario_064,snap,0.0,
@@ -1063,7 +1063,7 @@ scenario_067,payroll_tax,6300.5400390625,
scenario_067,self_employment_tax,0.0,
scenario_067,state_income_tax_before_refundable_credits,2486.908935546875,
scenario_067,state_refundable_credits,0.0,
-scenario_067,local_income_tax,0.0,
+scenario_067,local_income_tax,1348.83203125,
scenario_067,snap,0.0,
scenario_067,ssi,0.0,
scenario_067,tanf,0.0,
@@ -1201,7 +1201,7 @@ scenario_076,federal_income_tax_before_refundable_credits,19611.80078125,
scenario_076,federal_refundable_credits,0.0,
scenario_076,payroll_tax,12240.0,
scenario_076,self_employment_tax,0.0,
-scenario_076,state_income_tax_before_refundable_credits,6818.34423828125,
+scenario_076,state_income_tax_before_refundable_credits,6828.34423828125,
scenario_076,state_refundable_credits,465.0,
scenario_076,local_income_tax,0.0,
scenario_076,snap,0.0,
@@ -1309,12 +1309,12 @@ scenario_081,head_chip_eligible,0.0,0.0
scenario_081,head_medicare_eligible,0.0,0.0
scenario_081,free_school_meals_eligible,0.0,0.0
scenario_081,reduced_price_school_meals_eligible,0.0,0.0
-scenario_082,federal_income_tax_before_refundable_credits,9563.052734375,
+scenario_082,federal_income_tax_before_refundable_credits,9579.767578125,
scenario_082,federal_refundable_credits,0.0,
scenario_082,payroll_tax,8108.02734375,
scenario_082,self_employment_tax,0.0,
-scenario_082,state_income_tax_before_refundable_credits,5598.55224609375,
-scenario_082,state_refundable_credits,667.0,
+scenario_082,state_income_tax_before_refundable_credits,5599.1435546875,
+scenario_082,state_refundable_credits,1187.5478515625,
scenario_082,local_income_tax,0.0,
scenario_082,snap,0.0,
scenario_082,ssi,0.0,
@@ -1363,7 +1363,7 @@ scenario_084,head_chip_eligible,0.0,0.0
scenario_084,head_medicare_eligible,0.0,0.0
scenario_084,free_school_meals_eligible,0.0,0.0
scenario_084,reduced_price_school_meals_eligible,0.0,0.0
-scenario_085,federal_income_tax_before_refundable_credits,2322.760009765625,
+scenario_085,federal_income_tax_before_refundable_credits,2322.8486328125,
scenario_085,federal_refundable_credits,0.0,
scenario_085,payroll_tax,0.0,
scenario_085,self_employment_tax,0.0,
@@ -1541,7 +1541,7 @@ scenario_098,head_chip_eligible,0.0,0.0
scenario_098,head_medicare_eligible,1.0,5285.2001953125
scenario_098,free_school_meals_eligible,0.0,0.0
scenario_098,reduced_price_school_meals_eligible,0.0,0.0
-scenario_099,federal_income_tax_before_refundable_credits,10679.75,
+scenario_099,federal_income_tax_before_refundable_credits,10711.484375,
scenario_099,federal_refundable_credits,0.0,
scenario_099,payroll_tax,14767.5,
scenario_099,self_employment_tax,0.0,
@@ -1723,11 +1723,11 @@ scenario_109,child2_early_head_start_eligible,0.0,0.0
scenario_109,child3_early_head_start_eligible,1.0,21977.013671875
scenario_109,free_school_meals_eligible,1.0,2261.924560546875
scenario_109,reduced_price_school_meals_eligible,0.0,0.0
-scenario_110,federal_income_tax_before_refundable_credits,23897.443359375,
+scenario_110,federal_income_tax_before_refundable_credits,25747.7734375,
scenario_110,federal_refundable_credits,0.0,
scenario_110,payroll_tax,7650.0,
scenario_110,self_employment_tax,0.0,
-scenario_110,state_income_tax_before_refundable_credits,4057.47216796875,
+scenario_110,state_income_tax_before_refundable_credits,4236.8076171875,
scenario_110,state_refundable_credits,0.0,
scenario_110,local_income_tax,0.0,
scenario_110,snap,0.0,
@@ -1913,7 +1913,7 @@ scenario_120,federal_income_tax_before_refundable_credits,40021.81640625,
scenario_120,federal_refundable_credits,0.0,
scenario_120,payroll_tax,13496.1552734375,
scenario_120,self_employment_tax,0.0,
-scenario_120,state_income_tax_before_refundable_credits,11917.18359375,
+scenario_120,state_income_tax_before_refundable_credits,11919.0576171875,
scenario_120,state_refundable_credits,0.0,
scenario_120,local_income_tax,0.0,
scenario_120,snap,0.0,
diff --git a/policybench/fix_module_closure.py b/policybench/fix_module_closure.py
new file mode 100644
index 00000000..d46d6325
--- /dev/null
+++ b/policybench/fix_module_closure.py
@@ -0,0 +1,252 @@
+"""The local files an audited fix module loads.
+
+A fix module in ``reference_audit/2026-09-22/fixes`` can load a sibling file,
+which then runs (or is read) with it, so a pin on the module must also pin
+everything it loads. The one supported loader is
+``Path(__file__).with_name("")`` with a literal file name. A module that
+reaches local files another way has a closure these gates cannot establish,
+so it is refused rather than pinned short. The other ways are:
+
+- another use of ``__file__`` (``Path(__file__).resolve().parent / f"{name}.py"``);
+- a file name given outside that call (``open("x.json")``, ``_HERE / "x.py"``);
+- ``sys.path``, ``runpy``, ``importlib.import_module``, ``__import__``, a
+ file loader class, ``os.chdir``, or the ``exec``/``eval``/``compile``
+ builtins;
+- listing a directory (``glob``, ``rglob``, ``iterdir``, ``listdir``,
+ ``scandir``, ``walk``), which turns a pattern into files no literal names;
+- reaching the module's own location another way (``__spec__``,
+ ``__loader__``, an attribute ``.__file__`` such as
+ ``sys.modules[__name__].__file__``, ``inspect``, ``globals()``, ``vars()``);
+- importing this repository's own package (``policybench``), whose code no
+ fix-module pin covers;
+- building a path from anything but ``__file__`` (``Path(stem + ".py")``,
+ ``Path.cwd() / name``, ``os.getcwd()``, ``Path.home()``), or opening a file
+ whose name is not a literal (``open(os.environ["X"])``).
+
+What this cannot see is code that reaches files only through names it
+computes at run time from data (a string read from a file, say); every
+module the engine-upgrade evidence applies is checked by a test to load
+its siblings only in the supported form.
+
+The builder (``reference_audit/2026-10-09-engine-upgrade/scripts/
+build_references_upgrade.py``) and the release driver
+(``scripts/finish_haiku55.py``) both derive a target's pinned siblings here,
+from the committed bytes.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from collections.abc import Callable
+
+# A literal sibling file name: no directory part.
+SIBLING = re.compile(r"^[A-Za-z0-9_.\-]+\.[A-Za-z0-9]+$")
+# A string constant that names a local file.
+FILE_NAME = re.compile(
+ r"^[\w.\-/]*\.(py|pyc|json|csv|tsv|ya?ml|toml|txt|pkl|pickle|parquet|h5|npz|npy)$",
+ re.IGNORECASE,
+)
+# Builtins that run or load code a static read cannot follow.
+BANNED_BUILTINS = frozenset(
+ {"__import__", "exec", "eval", "compile", "globals", "vars"}
+)
+# Names, as a bare name or an attribute, that load code or move the paths
+# local files resolve against.
+BANNED_NAMES = frozenset(
+ {
+ "import_module",
+ "run_path",
+ "run_module",
+ "runpy",
+ "SourceFileLoader",
+ "SourcelessFileLoader",
+ "ExtensionFileLoader",
+ "chdir",
+ "getfile",
+ "getsourcefile",
+ "glob",
+ "rglob",
+ "iterdir",
+ "listdir",
+ "scandir",
+ "walk",
+ "__spec__",
+ "__loader__",
+ "cwd",
+ "getcwd",
+ "home",
+ "expanduser",
+ }
+)
+# Modules whose import reaches code or files a fix-module pin does not cover:
+# runpy runs other files, inspect finds them, and policybench is this
+# repository's own package.
+BANNED_IMPORTS = frozenset({"runpy", "inspect", "policybench"})
+
+
+class ClosureError(ValueError):
+ """A fix module whose local dependency closure cannot be established."""
+
+
+def _docstrings(tree: ast.AST) -> set[int]:
+ found = set()
+ for node in ast.walk(tree):
+ if isinstance(
+ node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+ ):
+ body = node.body
+ if (
+ body
+ and isinstance(body[0], ast.Expr)
+ and isinstance(body[0].value, ast.Constant)
+ and isinstance(body[0].value.value, str)
+ ):
+ found.add(id(body[0].value))
+ return found
+
+
+def _is_path_call(node: ast.AST) -> bool:
+ func = node.func if isinstance(node, ast.Call) else None
+ return (isinstance(func, ast.Name) and func.id == "Path") or (
+ isinstance(func, ast.Attribute) and func.attr == "Path"
+ )
+
+
+def direct_dependencies(text: str, name: str = "") -> list[str]:
+ """The sibling files one fix module's source loads, in source order.
+ Raises ClosureError when the module reaches local files any other way."""
+ try:
+ tree = ast.parse(text)
+ except SyntaxError as error:
+ raise ClosureError(f"{name} does not parse: {error}") from None
+ parents: dict[int, ast.AST] = {}
+ for node in ast.walk(tree):
+ for child in ast.iter_child_nodes(node):
+ parents[id(child)] = node
+ docstrings = _docstrings(tree)
+ literals: set[int] = set()
+ found: list[tuple[int, int, str]] = []
+ problems: list[str] = []
+
+ def at(node) -> str:
+ return f"{name}:{getattr(node, 'lineno', '?')}"
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Name) and node.id == "__file__":
+ call = parents.get(id(node))
+ attr = parents.get(id(call)) if call is not None else None
+ outer = parents.get(id(attr)) if attr is not None else None
+ if not (
+ _is_path_call(call)
+ and call.args == [node]
+ and not call.keywords
+ and isinstance(attr, ast.Attribute)
+ and attr.value is call
+ and attr.attr == "with_name"
+ and isinstance(outer, ast.Call)
+ and outer.func is attr
+ and len(outer.args) == 1
+ and not outer.keywords
+ and isinstance(outer.args[0], ast.Constant)
+ and isinstance(outer.args[0].value, str)
+ and SIBLING.match(outer.args[0].value)
+ and ".." not in outer.args[0].value
+ ):
+ problems.append(
+ f"{at(node)} uses __file__ other than as "
+ 'Path(__file__).with_name("")'
+ )
+ continue
+ literal = outer.args[0]
+ literals.add(id(literal))
+ found.append((literal.lineno, literal.col_offset, literal.value))
+ elif (
+ isinstance(node, ast.Call)
+ and _is_path_call(node)
+ and not (
+ len(node.args) == 1
+ and isinstance(node.args[0], ast.Name)
+ and node.args[0].id == "__file__"
+ )
+ ):
+ problems.append(f"{at(node)} builds a Path from something but __file__")
+ elif (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "open"
+ and not (node.args and isinstance(node.args[0], ast.Constant))
+ ):
+ problems.append(f"{at(node)} opens a file whose name is not a literal")
+ elif isinstance(node, ast.Name) and node.id in BANNED_BUILTINS:
+ problems.append(f"{at(node)} calls {node.id}")
+ elif isinstance(node, ast.Name) and node.id in BANNED_NAMES:
+ problems.append(f"{at(node)} uses {node.id}")
+ elif isinstance(node, ast.Attribute) and node.attr in BANNED_NAMES:
+ problems.append(f"{at(node)} uses .{node.attr}")
+ elif isinstance(node, ast.Attribute) and node.attr == "__file__":
+ problems.append(f"{at(node)} uses .__file__ of another object")
+ elif (
+ isinstance(node, ast.Attribute)
+ and node.attr == "path"
+ and isinstance(node.value, ast.Name)
+ and node.value.id == "sys"
+ ):
+ problems.append(f"{at(node)} uses sys.path")
+ elif isinstance(node, (ast.Import, ast.ImportFrom)):
+ modules = (
+ [alias.name for alias in node.names]
+ if isinstance(node, ast.Import)
+ else [node.module or ""]
+ )
+ imported = {alias.name for alias in node.names}
+ if any(m.split(".")[0] in BANNED_IMPORTS for m in modules) or (
+ isinstance(node, ast.ImportFrom) and node.level
+ ):
+ problems.append(
+ f"{at(node)} imports {modules} relatively or from "
+ f"{sorted(BANNED_IMPORTS)}"
+ )
+ elif isinstance(node, ast.ImportFrom) and (
+ banned := imported & (BANNED_NAMES | BANNED_BUILTINS)
+ ):
+ problems.append(f"{at(node)} imports {sorted(banned)}")
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Constant)
+ and isinstance(node.value, str)
+ and id(node) not in literals
+ and id(node) not in docstrings
+ and FILE_NAME.match(node.value.strip())
+ ):
+ problems.append(
+ f"{at(node)} names the file {node.value!r} outside "
+ 'Path(__file__).with_name("")'
+ )
+ if problems:
+ raise ClosureError(
+ f"the local files {name} loads cannot be established: {'; '.join(problems)}"
+ )
+ ordered: list[str] = []
+ for _, _, literal in sorted(found):
+ if literal not in ordered:
+ ordered.append(literal)
+ return ordered
+
+
+def dependency_closure(name: str, read: Callable[[str], bytes]) -> list[str]:
+ """Every sibling file the named fix module loads, transitively, in
+ discovery order, without the module itself; ``read(file)`` gives a file's
+ bytes (as committed). A loaded Python file is followed in turn; any other
+ file (a JSON table) is a leaf. Raises ClosureError for a module whose
+ closure cannot be established."""
+ found: list[str] = []
+ queue = [name] if name.endswith(".py") else []
+ while queue:
+ current = queue.pop(0)
+ for dep in direct_dependencies(read(current).decode(), current):
+ if dep != name and dep not in found:
+ found.append(dep)
+ if dep.endswith(".py"):
+ queue.append(dep)
+ return found
diff --git a/policybench/paper_results.py b/policybench/paper_results.py
index 663c36a3..7375450c 100644
--- a/policybench/paper_results.py
+++ b/policybench/paper_results.py
@@ -55,6 +55,11 @@
ROOT = Path(__file__).resolve().parents[1]
SNAPSHOT_DIR = ROOT / "paper" / "snapshot" / "20260501"
+# The 2026-09-29 move from policyengine-us 1.755.4 to 2.15.17. Its audit,
+# reference_audit/2026-09-28, holds the publication check, the sweep timing
+# and the re-run sweeps below; they describe that upgrade, whichever upgrades
+# follow it (the engine_upgrade_* accessors follow the last).
+SEPTEMBER_UPGRADE_DATE = "2026-09-29"
UPGRADE_VERIFICATION = ROOT / "reference_audit" / "2026-09-28" / "verification"
# The sweep that rechecked every reference on the newest policyengine-us
# release when PolicyBench checked PyPI before publishing, with the fix module
@@ -66,6 +71,58 @@
# What each exclusion sweep re-run on the reference engine moves.
RERUN_SWEEPS = UPGRADE_VERIFICATION / "rerun_sweeps.json"
+# The phrase an upgrade's recheck reason uses for an output whose recorded
+# defect the new engine fixes but whose reference rests on another one.
+FURTHER_DEFECT = "rests on a further engine defect"
+
+# The October 2026 engine move's audit: its builder, and the timing record and
+# publication check scripts/sweep_timing.py writes there.
+OCTOBER_UPGRADE_AUDIT = "reference_audit/2026-10-09-engine-upgrade"
+OCTOBER_SWEEP_TIMING = (
+ ROOT / OCTOBER_UPGRADE_AUDIT / "verification" / "sweep_timing.json"
+)
+
+# How the paper names each engine defect an upgrade can fix, after the audits'
+# own statements of them (reference_audit/2026-09-22/root_causes.json and
+# reference_audit/2026-10-05-reference-adversary/proposed_changes.json). An
+# upgrade that restores an output whose root cause has no name here stops the
+# render, so a new fix is named before it is published.
+ROOT_CAUSE_LABELS = {
+ "az_standard_deduction_indexing": "Arizona's standard deduction indexing",
+ "oh_medical_deduction_premiums": (
+ "Ohio's medical deduction for health insurance premiums"
+ ),
+ "co_sales_tax_refund_surplus": "Colorado's 2026 sales tax refund",
+ "ny_cdcc_606_c2": "New York's 2026 child and dependent care credit",
+ "r01_ira_compensation": (
+ "the IRA deduction's compensation limit and a dependent's contributions"
+ ),
+ "r02_ira_219g": "the IRA deduction's active-participant phase-out",
+ "r03_estate_income": "estate income in gross income",
+ "r05_nj_worker_ui": "New Jersey's worker unemployment and workforce contributions",
+ "r06_wi_act15_before_refundable": (
+ "Wisconsin's retirement income exclusion before refundable credits"
+ ),
+ "r07_idaho_health_premiums": "Idaho's subtraction for health insurance premiums",
+ "r08_eitc_earned_income_deferrals": (
+ "elective deferrals in the earned income behind the EITC"
+ ),
+ "r11_ca_itemized_conformity": "California's itemized deduction conformity",
+ "r22_ma_part_a_loss_offset": "Massachusetts's Part A capital loss offset",
+ "r30_snap_heat_and_eat_sua": "the heat-and-eat SNAP utility allowance",
+ "r32_wi_capital_gain_distributions": (
+ "Wisconsin's capital gain subtraction for distributions"
+ ),
+}
+# The same for the cause of a scored reference an upgrade changes.
+CHANGE_CAUSE_LABELS = {
+ "id_permanent_building_fund_tax": (
+ "Idaho's $10 permanent building fund tax (Idaho Code 63-3082), which "
+ "policyengine-us now counts in state income tax"
+ ),
+}
+UPSTREAM_PR = re.compile(r"policyengine-us#(\d+)")
+
# The 2026-10-05 review of release dashboard-data-20260930. Each of its three
# audits recomputed every output on the reference engine under readings of an
# input the prompt never states. Each entry names the sweep's CSV, the column
@@ -73,6 +130,9 @@
# columns holding the readings and checks it ran. The records the review added
# carry ``decided_on`` = REVIEW_DATE.
REVIEW_DATE = "2026-10-05"
+# The day Max ruled on the reference adversary's records (d1022) and the
+# Louisiana records (d994); each such record names its ruling in ``decision``.
+RULING_DATE = "2026-10-06"
REVIEW_SWEEPS: dict[str, tuple[Path, str, tuple[str, ...]]] = {
# State income tax in the federal SALT deduction
# (reference_audit/2026-10-05/README.md).
@@ -124,6 +184,27 @@
}
+def _series(parts: list[str]) -> str:
+ """'a', 'a and b', or 'a, b and c'."""
+ if len(parts) <= 2:
+ return " and ".join(parts)
+ return ", ".join(parts[:-1]) + " and " + parts[-1]
+
+
+def uploaded_phrase(uploaded: str, day: str) -> str:
+ """When a wheel went up, as the timing sentences state it: the clock time,
+ and the date too when the upload was not on ``day`` (the sweep's)."""
+ clock = uploaded[11:16]
+ if uploaded[:10] == day:
+ return f"uploaded {clock} UTC"
+ return f"uploaded {uploaded[:10]} at {clock} UTC"
+
+
+def count_word(n: int) -> str:
+ """A count as the paper writes it: a word up to ten, digits above."""
+ return NUMBER_WORDS.get(n, f"{n:,}")
+
+
def moves_beyond_tolerance(variable: str, before: float, after: float) -> bool:
"""Whether a value change leaves the exact-match tolerance: more than $1
for an amount output, any change for a 0/1 flag."""
@@ -200,24 +281,37 @@ def partition_review_sweep_moves(
def partition_engine_upgrade_changes(
- changes: list[dict], excluded: frozenset[tuple[str, str]] | set[tuple[str, str]]
+ changes: list[dict],
+ excluded: frozenset[tuple[str, str]] | set[tuple[str, str]],
+ restored: frozenset[tuple[str, str]] | set[tuple[str, str]] = frozenset(),
) -> dict[str, list[dict]]:
- """Split an engine upgrade's changed outputs into three disjoint groups.
-
- A change to an output the exclusion record now lists is a new exclusion.
- A scored change moves beyond the exact-match tolerance ($1 for an amount,
- any change for a 0/1 flag) or within it. Every change lands in exactly one
- group, so the three counts add up to the revision's changed list.
+ """Split an engine upgrade's changed outputs into four disjoint groups.
+
+ ``excluded`` is the exclusion record once the upgrade was decided, and
+ ``restored`` the excluded outputs the upgrade returned to scoring at the
+ new engine's values. A change to an excluded output is a new exclusion; a
+ change to a restored output is a restoration. Any other change is scored,
+ and moves beyond the exact-match tolerance ($1 for an amount, any change
+ for a 0/1 flag) or within it. Every change lands in exactly one group, so
+ the four counts add up to the revision's changed list.
"""
+ both = set(excluded) & set(restored)
+ if both:
+ raise ValueError(f"outputs both excluded and restored: {sorted(both)}")
partition: dict[str, list[dict]] = {
"scored_changes": [],
"within_tolerance": [],
"new_exclusions": [],
+ "restored": [],
}
for change in changes:
- if (change["scenario_id"], change["variable"]) in excluded:
+ key = (change["scenario_id"], change["variable"])
+ if key in excluded:
partition["new_exclusions"].append(change)
continue
+ if key in restored:
+ partition["restored"].append(change)
+ continue
moved = abs(change["regenerated"] - change["previous"])
amount = metric_type_for_output(change["variable"]) == "amount"
beyond = moved > 1 if amount else moved > 0
@@ -231,6 +325,254 @@ def partition_engine_upgrade_changes(
return partition
+def engine_version_number(label: str) -> str:
+ """'policyengine-us 2.15.17' -> '2.15.17' (a bare version is returned as is)."""
+ return label.removeprefix("policyengine-us ")
+
+
+def engine_version_key(version: str) -> tuple[int, ...]:
+ """Numeric sort key of a policyengine-us release, so 2.4.0 < 2.15.17."""
+ return tuple(int(part) for part in engine_version_number(version).split("."))
+
+
+def engine_version_count_phrase(counts: dict[str, int]) -> str:
+ """'52 computed with policyengine-us 1.755.4, 12 with 2.15.17': one clause
+ per engine, oldest release first, for any number of engines."""
+ ordered = sorted(counts, key=engine_version_key)
+ clauses = [f"{counts[version]:,} with {version}" for version in ordered]
+ if clauses:
+ clauses[0] = (
+ f"{counts[ordered[0]]:,} computed with policyengine-us {ordered[0]}"
+ )
+ return ", ".join(clauses)
+
+
+def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str:
+ """Two sentences on the outputs an engine upgrade fixes: ``restored - ruled``
+ that earlier releases excluded return to scoring at the new version's
+ values, and ``ruled`` that rulings on ``ruling_date`` decided while they
+ were still scored stay scored at them."""
+ if not 0 <= ruled <= restored:
+ raise ValueError(f"{ruled} ruled of {restored} restored outputs")
+ if not restored:
+ return ""
+ returned = restored - ruled
+ defects = "defect" if restored == 1 else "defects"
+ noun = "output" if restored == 1 else "outputs"
+ lead = f"The move fixes the engine {defects} behind {count_word(restored)} {noun}."
+ if not ruled:
+ if restored == 1:
+ return (
+ f"{lead} An earlier release excluded it, and it returns to "
+ "scoring at the new version's values."
+ )
+ return (
+ f"{lead} Earlier releases excluded them, and they return to scoring "
+ "at the new version's values."
+ )
+ if not returned:
+ them, were, stay = (
+ ("it", "it was", "it stays")
+ if ruled == 1
+ else ("them", "they were", "they stay")
+ )
+ return (
+ f"{lead} The {ruling_date} rulings decided {them} while {were} still "
+ f"scored, and {stay} scored at the new version's values."
+ )
+ which = "which returns" if returned == 1 else "which return"
+ stays = "stays" if ruled == 1 else "stay"
+ return (
+ f"{lead} Earlier releases excluded {count_word(returned)} of them, {which} "
+ f"to scoring at the new version's values; the other {count_word(ruled)}, "
+ f"ruled on {ruling_date} while still scored, {stays} scored at those values."
+ )
+
+
+def _key(entry: dict) -> tuple[str, str]:
+ return entry["scenario_id"], entry.get("variable", "snap")
+
+
+def exclusions_in_place(
+ exclusions: list[dict], revisions: list[dict], date: str, *, include_day: bool
+) -> list[dict]:
+ """The exclusion records in place just before ``date`` (UTC day) or, with
+ ``include_day``, at its end: records decided by then and not yet removed.
+
+ ``exclusions`` is the current record. A revision that returns excluded
+ outputs to scoring lists each in ``regenerated_exclusions`` with the
+ record it removes (``record``), so the record as it stood at any earlier
+ date can be rebuilt: a record a later revision removed was still in place.
+ """
+
+ def by_then(day: str) -> bool:
+ return day <= date if include_day else day < date
+
+ history = [(record, None) for record in exclusions] + [
+ (entry["record"], revision["date"])
+ for revision in revisions
+ for entry in revision.get("regenerated_exclusions", [])
+ ]
+ return [
+ record
+ for record, removed_on in history
+ if by_then(record["decided_on"])
+ and (removed_on is None or not by_then(removed_on))
+ ]
+
+
+class EngineUpgrade:
+ """One ``engine_upgrade`` revision of the reference sidecar, with the
+ exclusion record as it stood around it.
+
+ ``excluded_before`` holds the outputs excluded when the upgrade began:
+ records decided before its date and not removed before it. ``excluded``
+ holds those excluded once it was decided: records decided by the end of
+ its date and not removed by then. A record decided after the upgrade never
+ counts here, and a record a later revision removed still does.
+ ``restored`` holds the outputs the upgrade returned to scoring (its
+ ``regenerated_exclusions``).
+ """
+
+ def __init__(
+ self,
+ revision: dict,
+ excluded_before: frozenset[tuple[str, str]],
+ excluded: frozenset[tuple[str, str]],
+ ):
+ self.revision = revision
+ self.excluded_before = excluded_before
+ self.excluded = excluded
+ self.restored = frozenset(
+ _key(entry) for entry in revision.get("regenerated_exclusions", [])
+ )
+
+ def __repr__(self) -> str:
+ return (
+ f"EngineUpgrade({self.date}: {self.previous_engine_version} -> "
+ f"{self.engine_version})"
+ )
+
+ @property
+ def date(self) -> str:
+ return self.revision["date"]
+
+ @property
+ def engine_version(self) -> str:
+ return engine_version_number(self.revision["engine_version"])
+
+ @property
+ def previous_engine_version(self) -> str:
+ return engine_version_number(self.revision["previous_engine_version"])
+
+ @property
+ def changes(self) -> list[dict]:
+ return self.revision["changed"]
+
+ @cached_property
+ def partition(self) -> dict[str, list[dict]]:
+ """The changed outputs, split into scored changes beyond the
+ exact-match tolerance, scored changes within it, new exclusions and
+ restorations. Refuses a change to an output excluded both before and
+ after the upgrade: an excluded output keeps the value it was decided
+ on (rule 5)."""
+ kept = (self.excluded_before & self.excluded) - self.restored
+ touched = sorted(kept & {_key(change) for change in self.changes})
+ if touched:
+ raise ValueError(
+ f"the {self.date} upgrade changes outputs it kept excluded: {touched}"
+ )
+ return partition_engine_upgrade_changes(
+ self.changes, self.excluded - self.excluded_before, self.restored
+ )
+
+ @property
+ def scored_change_count(self) -> int:
+ return len(self.partition["scored_changes"])
+
+ @property
+ def within_tolerance_count(self) -> int:
+ return len(self.partition["within_tolerance"])
+
+ @property
+ def new_exclusion_count(self) -> int:
+ return len(self.partition["new_exclusions"])
+
+ @property
+ def restored_count(self) -> int:
+ return len(self.restored)
+
+ @property
+ def rechecked(self) -> list[dict]:
+ """Excluded outputs whose value moves on the new engine; each stays
+ excluded with the value it was decided on."""
+ return self.revision.get("excluded_outputs_rechecked", [])
+
+ @property
+ def rechecked_count(self) -> int:
+ return len(self.rechecked)
+
+ def rechecked_value(self, record: dict) -> float:
+ """A rechecked output's value on the new engine (``value_on_``)."""
+ return record["value_on_" + self.engine_version.replace(".", "_")]
+
+
+def engine_upgrades_from(
+ revisions: list[dict], exclusions: list[dict]
+) -> list[EngineUpgrade]:
+ """The sidecar's engine upgrades, oldest first, each with the exclusion
+ record as it stood around it (``exclusions_in_place``). Refuses a chain
+ in which an upgrade does not start from the engine the one before it
+ moved to."""
+
+ def in_place(date: str, include_day: bool) -> frozenset[tuple[str, str]]:
+ return frozenset(
+ _key(record)
+ for record in exclusions_in_place(
+ exclusions, revisions, date, include_day=include_day
+ )
+ )
+
+ upgrades = [
+ EngineUpgrade(
+ revision,
+ excluded_before=in_place(revision["date"], False),
+ excluded=in_place(revision["date"], True),
+ )
+ for revision in revisions
+ if revision.get("kind") == "engine_upgrade"
+ ]
+ for before, after in zip(upgrades, upgrades[1:]):
+ if after.previous_engine_version != before.engine_version:
+ raise ValueError(
+ f"the {after.date} upgrade does not start from "
+ f"{before.engine_version}, the engine the {before.date} upgrade "
+ f"moved to: {after.previous_engine_version}"
+ )
+ return upgrades
+
+
+def references_as_of(
+ references: dict[tuple[str, str], float], revisions: list[dict], date: str
+) -> dict[tuple[str, str], float]:
+ """The references as they stood at the end of ``date``: ``references``
+ with every later revision's changes undone, newest first. Refuses a change
+ whose regenerated value is not where the later references stand."""
+ values = dict(references)
+ for revision in reversed(revisions):
+ if revision["date"] <= date:
+ continue
+ for change in reversed(revision.get("changed", [])):
+ key = _key(change)
+ if abs(values[key] - change["regenerated"]) > 1e-6:
+ raise ValueError(
+ f"{key}: the {revision['date']} revision does not end at "
+ f"the later reference ({change['regenerated']} != {values[key]})"
+ )
+ values[key] = change["previous"]
+ return values
+
+
# Human-readable model names for the frozen roster. Aliases that do not
# appear here fall back to a humanized form of the PolicyBench id.
MODEL_DISPLAY_NAMES = {
@@ -271,6 +613,7 @@ def partition_engine_upgrade_changes(
"claude-opus-4.8": "Claude Opus 4.8",
"claude-opus-4.7": "Claude Opus 4.7",
"claude-sonnet-4.6": "Claude Sonnet 4.6",
+ "claude-haiku-5.5": "Claude Haiku 5.5",
"claude-haiku-4.5": "Claude Haiku 4.5",
"grok-4.3": "Grok 4.3",
"gpt-5.5": "GPT-5.5",
@@ -1229,6 +1572,10 @@ def unlisted_input_exclusion_count(self) -> int:
def engine_defect_exclusion_count(self) -> int:
return sum(self.excluded_outputs_by_root_cause.values())
+ @property
+ def later_law_exclusion_count_word(self) -> str:
+ return count_word(self.later_law_exclusion_count)
+
@property
def excluded_output_households_fmt(self) -> str:
return f"{len({e['scenario_id'] for e in self.reference_exclusions}):,}"
@@ -1373,8 +1720,9 @@ def regenerated_reference_keys(self) -> set[tuple[str, str]]:
"""Scored outputs a convention or an upstream fix regenerated.
These are the September 22 regenerations, made on the engine version
- before the upgrade; the engine upgrade's own changes are counted by the
- ``engine_upgrade_*`` properties.
+ before the first upgrade; each engine upgrade's own changes are counted
+ by ``engine_upgrades`` (the last one's by the ``engine_upgrade_*``
+ properties).
"""
return {
(change["scenario_id"], change.get("variable", "snap"))
@@ -1384,70 +1732,412 @@ def regenerated_reference_keys(self) -> set[tuple[str, str]]:
}
@cached_property
+ def frozen_references(self) -> dict[tuple[str, str], float]:
+ """The frozen reference for every output, scored or excluded."""
+ run_dir = SNAPSHOT_DIR / "runs" / self.us_run_label
+ frozen = pd.read_csv(run_dir / "reference_outputs.csv")
+ return {
+ (row.scenario_id, row.variable): float(row.value)
+ for row in frozen.itertuples(index=False)
+ }
+
+ def references_as_of(self, date: str) -> dict[tuple[str, str], float]:
+ """The references as they stood at the end of ``date`` (UTC day)."""
+ return references_as_of(self.frozen_references, self.reference_revisions, date)
+
+ def _exclusions_in_place(self, date: str, *, include_day: bool) -> list[dict]:
+ """Exclusion records in place just before ``date`` or, with
+ ``include_day``, at its end, whatever later releases added or removed."""
+ return exclusions_in_place(
+ self.reference_exclusions,
+ self.reference_revisions,
+ date,
+ include_day=include_day,
+ )
+
+ @cached_property
+ def engine_upgrades(self) -> list[EngineUpgrade]:
+ """Every engine upgrade the sidecar records, oldest first."""
+ return engine_upgrades_from(self.reference_revisions, self.reference_exclusions)
+
+ @property
+ def engine_upgrade_count(self) -> int:
+ return len(self.engine_upgrades)
+
+ @property
+ def last_engine_upgrade(self) -> EngineUpgrade | None:
+ """The upgrade behind the current references."""
+ return self.engine_upgrades[-1] if self.engine_upgrades else None
+
+ def engine_upgrade_on(self, date: str) -> EngineUpgrade:
+ """The engine upgrade dated ``date`` (UTC day)."""
+ matches = [u for u in self.engine_upgrades if u.date == date]
+ if len(matches) != 1:
+ raise ValueError(
+ f"no engine_upgrade revision dated {date}, or more than one: "
+ f"{[u.date for u in self.engine_upgrades]}"
+ )
+ return matches[0]
+
+ def engine_upgrade_to(self, version: str) -> EngineUpgrade:
+ """The engine upgrade that moved the references to ``version``."""
+ version = engine_version_number(version)
+ matches = [u for u in self.engine_upgrades if u.engine_version == version]
+ if len(matches) != 1:
+ raise ValueError(
+ f"no engine_upgrade revision to {version}, or more than one: "
+ f"{[u.engine_version for u in self.engine_upgrades]}"
+ )
+ return matches[0]
+
+ @property
+ def september_upgrade(self) -> EngineUpgrade:
+ """The 2026-09-29 move from policyengine-us 1.755.4 to 2.15.17, which
+ reference_audit/2026-09-28 records, however many upgrades follow."""
+ return self.engine_upgrade_on(SEPTEMBER_UPGRADE_DATE)
+
+ @property
def engine_upgrade_revision(self) -> dict | None:
- """The sidecar revision that moved the references to a newer engine."""
- upgrades = [
- revision
- for revision in self.reference_revisions
- if revision.get("kind") == "engine_upgrade"
- ]
- return upgrades[-1] if upgrades else None
+ """The sidecar revision that moved the references to the current
+ engine: the last engine upgrade."""
+ last = self.last_engine_upgrade
+ return None if last is None else last.revision
@property
def previous_policyengine_us_version(self) -> str:
- """policyengine-us version behind the references before the upgrade,
- the version the September 22 audit ran on."""
- revision = self.engine_upgrade_revision
- if revision is None:
- return self.policyengine_us_version
- return revision["previous_engine_version"].removeprefix("policyengine-us ")
+ """policyengine-us version behind the references before the last
+ upgrade: that revision's own previous_engine_version."""
+ last = self.last_engine_upgrade
+ return (
+ self.policyengine_us_version
+ if last is None
+ else (last.previous_engine_version)
+ )
@property
def engine_upgrade_date(self) -> str:
- """UTC date PolicyBench rebuilt the references on the new engine: the
- sidecar's ``regenerated_at_utc`` day, which is also the revision's own
- ``date`` (tests/test_paper_results.py checks they agree)."""
- if self.engine_upgrade_revision is None:
+ """UTC date PolicyBench rebuilt the references on the current engine:
+ the sidecar's ``regenerated_at_utc`` day, which is also the last
+ upgrade's own ``date`` (tests/test_paper_results.py checks they agree)."""
+ if self.last_engine_upgrade is None:
return ""
return self.reference_rebuilt_date
- def _engine_upgrade_changes(self) -> list[dict]:
- revision = self.engine_upgrade_revision
- return [] if revision is None else revision["changed"]
-
- @cached_property
+ @property
def engine_upgrade_partition(self) -> dict[str, list[dict]]:
- """The upgrade's changed outputs, split into scored changes beyond the
- exact-match tolerance, scored changes within it, and new exclusions."""
- return partition_engine_upgrade_changes(
- self._engine_upgrade_changes(), self._excluded_output_keys
- )
+ """The last upgrade's changed outputs, split into scored changes beyond
+ the exact-match tolerance, scored changes within it, new exclusions and
+ restorations."""
+ last = self.last_engine_upgrade
+ if last is None:
+ return partition_engine_upgrade_changes([], frozenset())
+ return last.partition
@property
def engine_upgrade_scored_change_count(self) -> int:
- """Scored references the upgrade moved beyond the exact-match tolerance."""
+ """Scored references the last upgrade moved beyond the exact-match
+ tolerance."""
return len(self.engine_upgrade_partition["scored_changes"])
@property
def engine_upgrade_within_tolerance_count(self) -> int:
- """Scored references the upgrade moved within the $1 tolerance."""
+ """Scored references the last upgrade moved within the $1 tolerance."""
return len(self.engine_upgrade_partition["within_tolerance"])
@property
def engine_upgrade_new_exclusion_count(self) -> int:
- """Outputs scored before the upgrade that it removed from scoring."""
+ """Outputs scored before the last upgrade that it removed from scoring."""
return len(self.engine_upgrade_partition["new_exclusions"])
+ @property
+ def engine_upgrade_restored_count(self) -> int:
+ """Excluded outputs the last upgrade returned to scoring."""
+ last = self.last_engine_upgrade
+ return 0 if last is None else last.restored_count
+
+ @property
+ def engine_upgrade_restored_count_word(self) -> str:
+ return count_word(self.engine_upgrade_restored_count)
+
+ @property
+ def engine_upgrade_restored_ruled_count(self) -> int:
+ """Restored outputs the 2026-10-06 rulings had decided to exclude. No
+ published board excluded them: they were still scored when the rulings
+ came, so they stay scored rather than return to scoring."""
+ last = self.last_engine_upgrade
+ if last is None:
+ return 0
+ ruled = {_key(record) for record in self.ruled_records}
+ return sum(
+ _key(entry) in ruled for entry in last.revision["regenerated_exclusions"]
+ )
+
+ @property
+ def engine_upgrade_restored_split_sentence(self) -> str:
+ """The last upgrade's fixed outputs, split by whether a published board
+ had excluded them (they return to scoring) or the 2026-10-06 rulings
+ decided them while they were still scored (they stay scored)."""
+ return restored_split_sentence(
+ self.engine_upgrade_restored_count,
+ self.engine_upgrade_restored_ruled_count,
+ RULING_DATE,
+ )
+
+ @cached_property
+ def engine_upgrade_timing(self) -> dict | None:
+ """The last upgrade's sweep timing record, when that upgrade is the
+ October 2026 audit's build and the record is written (None before)."""
+ last = self.last_engine_upgrade
+ if last is None or not str(last.revision.get("builder", "")).startswith(
+ OCTOBER_UPGRADE_AUDIT
+ ):
+ return None
+ if not OCTOBER_SWEEP_TIMING.is_file():
+ return None
+ timing = json.loads(OCTOBER_SWEEP_TIMING.read_text())
+ if timing["reference_sweep"]["engine"] != last.engine_version:
+ raise ValueError(
+ f"{OCTOBER_SWEEP_TIMING} times policyengine-us "
+ f"{timing['reference_sweep']['engine']}, not {last.engine_version}"
+ )
+ return timing
+
+ @property
+ def engine_upgrade_timing_sentence(self) -> str:
+ """When the last upgrade's sweep began and what the publication check
+ found, from its timing record; empty until the record is written."""
+ timing = self.engine_upgrade_timing
+ if timing is None:
+ return ""
+ engine = timing["reference_sweep"]["engine"]
+ began = timing["reference_sweep"]["first_output_at_utc"][:10]
+ uploaded = uploaded_phrase(
+ timing["pypi"]["wheel_uploaded_at_utc"][engine], began
+ )
+ read_at = timing["pypi"]["read_at_utc"]
+ check = timing.get("publication_check")
+ if check is None or check["engine"] != timing["pypi"]["newest_at_read"]:
+ raise ValueError(f"{OCTOBER_SWEEP_TIMING} records no publication check")
+ when = f"PolicyBench checked PyPI on {read_at[:10]} at {read_at[11:16]} UTC"
+ if check["engine"] == engine:
+ checked = f"It was still the newest release when {when}."
+ elif check["same"] == check["outputs"]:
+ checked = (
+ f"policyengine-us {check['engine']}, the newest release when {when}, "
+ f"gives the same value as {engine} for all {check['outputs']:,} "
+ "outputs under the same conventions and adapter."
+ )
+ elif check["scored_same"] == check["scored_outputs"]:
+ moved = len(check["excluded_differ"])
+ checked = (
+ f"policyengine-us {check['engine']}, the newest release when {when}, "
+ f"gives the same value as {engine} for all "
+ f"{check['scored_outputs']:,} scored outputs under the same "
+ f"conventions and adapter, and moves {count_word(moved)} of the "
+ "excluded outputs, which keep the values they were decided on."
+ )
+ else:
+ raise ValueError(
+ f"policyengine-us {check['engine']} moves scored outputs "
+ f"{check['scored_differ']}; state what the publication check found "
+ "before publishing"
+ )
+ return (
+ f"policyengine-us {engine} was the newest release when PolicyBench began "
+ f"sweeping the references on {began} ({uploaded}). {checked}"
+ )
+
+ @property
+ def engine_upgrade_restored_fixes(self) -> list[tuple[str, int, list[str]]]:
+ """The defects behind the outputs the last upgrade returned to scoring:
+ each one's name, how many outputs it restores and the upstream pull
+ requests its regenerated_exclusions entries name (not the related
+ ones), in the sidecar's order. A record with several root causes names
+ each."""
+ last = self.last_engine_upgrade
+ fixes: dict[str, tuple[int, list[str]]] = {}
+ for entry in [] if last is None else last.revision["regenerated_exclusions"]:
+ causes = entry["record"]["root_cause"].split("+")
+ unnamed = [c for c in causes if c not in ROOT_CAUSE_LABELS]
+ if unnamed:
+ raise ValueError(f"no paper name for the root causes {unnamed}")
+ label = " and ".join(ROOT_CAUSE_LABELS[c] for c in causes)
+ prs = UPSTREAM_PR.findall(entry["upstream"].split("; related")[0])
+ if not prs:
+ raise ValueError(f"{_key(entry)} names no upstream pull request")
+ count, known = fixes.get(label, (0, []))
+ fixes[label] = (count + 1, known + [p for p in prs if p not in known])
+ return [(label, count, prs) for label, (count, prs) in fixes.items()]
+
+ @property
+ def engine_upgrade_restored_fix_phrase(self) -> str:
+ """The fixes behind the restored outputs as one series, each with the
+ upstream pull requests it names and, past one, how many outputs."""
+ parts = []
+ for label, count, prs in self.engine_upgrade_restored_fixes:
+ numbers = ", ".join(f"#{pr}" for pr in prs)
+ outputs = "" if count == 1 else f"; {count_word(count)} outputs"
+ parts.append(f"{label} (policyengine-us {numbers}{outputs})")
+ return _series(parts)
+
+ @property
+ def engine_upgrade_restored_sentence(self) -> str:
+ """One sentence naming the fixes behind the restored outputs."""
+ phrase = self.engine_upgrade_restored_fix_phrase
+ if not phrase:
+ return ""
+ return f"The restored outputs take the upstream fixes for {phrase}."
+
+ @property
+ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:
+ """The last upgrade's restored outputs by how their corrected value is
+ known (the sidecar's ``target.kind``): ``record`` for the value the
+ exclusion record carries, ``fix_modules`` for the value the audit's
+ fix modules give on a later engine that still has the defect. Refuses,
+ as the builder's regeneration_target does, an entry whose engine value
+ is not within its tolerance of its target (a flag must equal it), a
+ tolerance above $1, a ``record`` target that is not the record's value,
+ and a ``fix_modules`` target whose modules did not move the output
+ beyond the exact-match tolerance on that engine (no defect there)."""
+ last = self.last_engine_upgrade
+ targets: dict[str, list[dict]] = {"record": [], "fix_modules": []}
+ for entry in [] if last is None else last.revision["regenerated_exclusions"]:
+ target, key = entry["target"], _key(entry)
+ if target["kind"] not in targets:
+ raise ValueError(f"{key} has an unknown target kind {target['kind']!r}")
+ tolerance = entry["tolerance"]
+ if not 0 <= tolerance <= 1:
+ raise ValueError(f"{key} is restored on a tolerance of {tolerance}")
+ variable = entry["variable"]
+ if abs(
+ entry["regenerated"] - target["value"]
+ ) > tolerance or moves_beyond_tolerance(
+ variable, target["value"], entry["regenerated"]
+ ):
+ raise ValueError(f"{key} is restored off its target")
+ if target["kind"] == "record":
+ if target["value"] != entry["record"]["alternative_value"]:
+ raise ValueError(f"{key}'s target is not its record's value")
+ elif not moves_beyond_tolerance(
+ variable, target["engine_value"], target["value"]
+ ):
+ raise ValueError(f"{key}'s fix modules move nothing there")
+ targets[target["kind"]].append(entry)
+ return targets
+
+ @property
+ def engine_upgrade_restored_target_sentence(self) -> str:
+ """How the restored outputs' corrected values are known: by the
+ exclusion record, or by the audit's fix modules on a later engine."""
+ targets = self.engine_upgrade_restored_targets
+ by_record, by_modules = targets["record"], targets["fix_modules"]
+ total = len(by_record) + len(by_modules)
+ if not total:
+ return ""
+ if not by_modules:
+ subject = "It lands" if total == 1 else "Each lands"
+ carries = "its exclusion record carries"
+ return f"{subject} within $1 of the corrected value {carries}."
+ engines = sorted({e["target"]["engine"] for e in by_modules})
+ if len(engines) != 1:
+ raise ValueError(f"fix-module targets name several engines: {engines}")
+ some = bool(by_record)
+ if len(by_modules) == 1:
+ modules = (
+ f"{'The other one' if some else 'It'} is held"
+ f"{' instead' if some else ''} to the value the audit's fix "
+ f"modules give on {engines[0]}, an engine that still has the "
+ "defect; it lands within $1 of it."
+ )
+ else:
+ subject = f"The other {count_word(len(by_modules))}" if some else "They"
+ modules = (
+ f"{subject} are held{' instead' if some else ''} to the values "
+ f"the audit's fix modules give on {engines[0]}, an engine that "
+ "still has the defects; each lands within $1 of its own."
+ )
+ if not some:
+ return modules
+ if len(by_record) == 1:
+ landing = "one lands within $1 of the corrected value its record carries"
+ else:
+ landing = (
+ f"{count_word(len(by_record))} land within $1 of the corrected "
+ "values their exclusion records carry"
+ )
+ return f"Of the {count_word(total)}, {landing}. {modules}"
+
+ @property
+ def engine_upgrade_scored_change_sentence(self) -> str:
+ """One sentence naming what the last upgrade changed in the scored
+ references beyond the tolerance."""
+ last = self.last_engine_upgrade
+ changes = [] if last is None else last.partition["scored_changes"]
+ parts = []
+ for change in changes:
+ cause = change["cause"]
+ if cause not in CHANGE_CAUSE_LABELS:
+ raise ValueError(f"no paper name for the change cause {cause!r}")
+ prs = UPSTREAM_PR.findall(change["basis"])
+ numbers = f" (policyengine-us #{prs[-1]})" if prs else ""
+ part = f"{CHANGE_CAUSE_LABELS[cause]}{numbers}"
+ if part not in parts:
+ parts.append(part)
+ if not parts:
+ return ""
+ noun = "reference follows" if len(changes) == 1 else "references follow"
+ return f"The changed scored {noun} {_series(parts)}."
+
+ @property
+ def engine_upgrade_new_exclusion_sentence(self) -> str:
+ """One sentence naming the inputs behind the outputs the last upgrade
+ removed from scoring."""
+ last = self.last_engine_upgrade
+ if last is None or not last.partition["new_exclusions"]:
+ return ""
+ added = {_key(change) for change in last.partition["new_exclusions"]}
+ inputs = []
+ for record in self.reference_exclusions:
+ if _key(record) in added:
+ text = record.get("unlisted_input")
+ if not text:
+ raise ValueError(f"{_key(record)} names no unlisted input")
+ if text not in inputs:
+ inputs.append(text)
+ count = len(added)
+ noun = "output that leaves" if count == 1 else "outputs that leave"
+ verb = "depends" if count == 1 else "depend"
+ return (
+ f"The {count_word(count)} {noun} scoring {verb} on an input no prompt "
+ f"states: {_series(inputs)}."
+ )
+
+ @property
+ def engine_upgrade_scored_change_count_word(self) -> str:
+ return count_word(self.engine_upgrade_scored_change_count)
+
+ @property
+ def engine_upgrade_new_exclusion_count_word(self) -> str:
+ return count_word(self.engine_upgrade_new_exclusion_count)
+
@property
def excluded_outputs_by_engine_version(self) -> dict[str, int]:
"""Excluded outputs by the policyengine-us version behind the value
- each keeps: the version its exclusion was decided on."""
- return dict(
- Counter(
- entry["engine_version"].removeprefix("policyengine-us ")
- for entry in self.reference_exclusions
- )
+ each keeps (the version its exclusion was decided on), oldest first."""
+ counts = Counter(
+ engine_version_number(entry["engine_version"])
+ for entry in self.reference_exclusions
)
+ return {
+ version: counts[version]
+ for version in sorted(counts, key=engine_version_key)
+ }
+
+ @property
+ def excluded_outputs_by_engine_version_phrase(self) -> str:
+ """'52 computed with policyengine-us 1.755.4, 12 with 2.15.17', for
+ however many engines the excluded outputs' values come from."""
+ return engine_version_count_phrase(self.excluded_outputs_by_engine_version)
@property
def excluded_outputs_on_previous_engine_count(self) -> int:
@@ -1478,9 +2168,12 @@ def sweep_timing(self) -> dict:
@property
def reference_engine_uploaded_utc(self) -> str:
- """PyPI upload time (UTC, HH:MM) of the reference engine's wheel."""
- uploaded = self.sweep_timing["pypi"]["wheel_uploaded_at_utc"]
- return uploaded[self.policyengine_us_version][11:16]
+ """PyPI upload time (UTC, HH:MM) of the wheel the 2026-09-29 reference
+ sweep began on (sweep_timing.json's reference_sweep engine, 2.15.17),
+ however many upgrades follow it."""
+ engine = self.sweep_timing["reference_sweep"]["engine"]
+ self.engine_upgrade_to(engine) # refuses a timing record of no upgrade
+ return self.sweep_timing["pypi"]["wheel_uploaded_at_utc"][engine][11:16]
@property
def publication_check_pypi_read_date(self) -> str:
@@ -1503,10 +2196,26 @@ def rerun_sweeps(self) -> dict:
"""reference_audit/2026-09-28/verification/rerun_sweeps.json."""
return json.loads(RERUN_SWEEPS.read_text())
+ @cached_property
+ def _rerun_sweep_upgrade(self) -> EngineUpgrade:
+ """The upgrade to the engine the re-run sweeps ran on (2.15.17)."""
+ return self.engine_upgrade_to(self.rerun_sweeps["engine"])
+
+ @cached_property
+ def _excluded_while_rerun_engine_was_reference(self) -> frozenset:
+ """Outputs excluded while the re-run sweeps' engine was the reference
+ engine, at the latest: the current record, or, once a later upgrade
+ moved the references on, the record just before that upgrade."""
+ upgrades = self.engine_upgrades
+ index = upgrades.index(self._rerun_sweep_upgrade)
+ if index + 1 == len(upgrades):
+ return self._excluded_output_keys
+ return upgrades[index + 1].excluded_before
+
@cached_property
def rerun_sweep_partition(self) -> dict[str, list[tuple[str, str, str]]]:
return partition_rerun_sweep_moves(
- self.rerun_sweeps, self._excluded_output_keys
+ self.rerun_sweeps, self._excluded_while_rerun_engine_was_reference
)
@property
@@ -1547,14 +2256,8 @@ def rerun_sweep_scored_within_tolerance_count_word(self) -> str:
def rerun_sweep_new_excluded_outputs(self) -> list[tuple[str, str]]:
"""Outputs that a sweep first run on the reference engine moves
beyond the exact-match tolerance, against its own baseline, and that
- the exclusion record did not hold before the upgrade (no exclusion
- decided on the previous engine)."""
- previous = f"policyengine-us {self.previous_policyengine_us_version}"
- held_before = {
- (entry["scenario_id"], entry["variable"])
- for entry in self.reference_exclusions
- if entry["engine_version"] == previous
- }
+ the exclusion record did not hold before that engine's upgrade."""
+ held_before = self._rerun_sweep_upgrade.excluded_before
return sorted(
{
(move["scenario_id"], move["variable"])
@@ -1577,12 +2280,18 @@ def rerun_sweep_new_excluded_count_word(self) -> str:
def review_date(self) -> str:
return REVIEW_DATE
+ @cached_property
+ def _excluded_at_review(self) -> list[dict]:
+ """Exclusion records in place once the review decided, whatever
+ records later releases added or removed."""
+ return self._exclusions_in_place(REVIEW_DATE, include_day=True)
+
@cached_property
def review_exclusions(self) -> list[dict]:
"""Exclusion records the 2026-10-05 review added."""
return [
entry
- for entry in self.reference_exclusions
+ for entry in self._excluded_at_review
if entry.get("decided_on") == REVIEW_DATE
]
@@ -1592,8 +2301,11 @@ def review_exclusion_keys(self) -> frozenset[tuple[str, str]]:
@property
def excluded_before_review_keys(self) -> frozenset[tuple[str, str]]:
- """Outputs excluded before the review: every record it did not add."""
- return self._excluded_output_keys - self.review_exclusion_keys
+ """Outputs excluded before the review: every record in place once it
+ decided that it did not add."""
+ return frozenset(exclusion_keys(self._excluded_at_review)) - (
+ self.review_exclusion_keys
+ )
@property
def review_new_exclusion_count(self) -> int:
@@ -1603,6 +2315,73 @@ def review_new_exclusion_count(self) -> int:
def review_new_exclusion_count_word(self) -> str:
return NUMBER_WORDS[self.review_new_exclusion_count]
+ @cached_property
+ def ruled_records(self) -> list[dict]:
+ """The records the 2026-10-06 rulings decided, in (scenario, variable)
+ order: those in the exclusion record, and those a later engine upgrade
+ regenerated, whose removed record its regenerated_exclusions keep."""
+ current = [
+ record
+ for record in self.reference_exclusions
+ if record.get("decided_on") == RULING_DATE
+ ]
+ removed = [
+ entry["record"]
+ for upgrade in self.engine_upgrades
+ for entry in upgrade.revision.get("regenerated_exclusions", [])
+ if entry.get("record", {}).get("decided_on") == RULING_DATE
+ ]
+ records = sorted(current + removed, key=_key)
+ unnamed = [_key(record) for record in records if not record.get("decision")]
+ if unnamed:
+ raise ValueError(f"2026-10-06 records name no ruling: {unnamed}")
+ return records
+
+ @property
+ def ruling_date(self) -> str:
+ return RULING_DATE
+
+ @property
+ def ruled_exclusion_count(self) -> int:
+ return len(self.ruled_records)
+
+ @property
+ def ruled_exclusion_count_word(self) -> str:
+ return count_word(self.ruled_exclusion_count)
+
+ def ruled_decision_count(self, decision: str) -> int:
+ """Records one ruling (d1022 or d994) decided."""
+ return sum(record["decision"] == decision for record in self.ruled_records)
+
+ def ruled_decision_count_word(self, decision: str) -> str:
+ return count_word(self.ruled_decision_count(decision))
+
+ @property
+ def ruled_regenerated_count(self) -> int:
+ """Ruled records a later engine upgrade regenerated."""
+ current = {_key(record) for record in self.reference_exclusions}
+ return sum(_key(record) not in current for record in self.ruled_records)
+
+ @property
+ def ruled_regenerated_count_word(self) -> str:
+ return count_word(self.ruled_regenerated_count)
+
+ @property
+ def ruled_kept_count(self) -> int:
+ return self.ruled_exclusion_count - self.ruled_regenerated_count
+
+ @property
+ def ruled_kept_count_word(self) -> str:
+ return count_word(self.ruled_kept_count)
+
+ @property
+ def ruled_exclusion_engine_version(self) -> str:
+ """policyengine-us version the ruled records were computed on."""
+ versions = {record["engine_version"] for record in self.ruled_records}
+ if len(versions) != 1:
+ raise ValueError(f"the 2026-10-06 records name several engines: {versions}")
+ return engine_version_number(versions.pop())
+
@property
def review_exclusion_engine_version(self) -> str:
"""policyengine-us version the review's records were computed on."""
@@ -1650,7 +2429,7 @@ def review_sweep_partition(self) -> dict[str, list[tuple[str, str, str]]]:
return partition_review_sweep_moves(
self._review_sweep_moves(),
self.excluded_before_review_keys,
- self._excluded_output_keys,
+ frozenset(exclusion_keys(self._excluded_at_review)),
)
def review_sweep_moved_outputs(self, sweep: str) -> list[tuple[str, str]]:
@@ -1729,13 +2508,9 @@ def review_still_scored_moves_phrase(self) -> str:
def review_scored_before_count(self) -> int:
"""References scored before the review, which every review sweep's
baseline reproduces (refuses a sweep whose baseline misses one, or
- whose reference column is not the frozen reference)."""
- run_dir = SNAPSHOT_DIR / "runs" / self.us_run_label
- frozen = pd.read_csv(run_dir / "reference_outputs.csv")
- reference = {
- (row.scenario_id, row.variable): float(row.value)
- for row in frozen.itertuples(index=False)
- }
+ whose reference column is not the frozen reference as it stood at the
+ review)."""
+ reference = self.references_as_of(REVIEW_DATE)
scored_before = set(reference) - self.excluded_before_review_keys
for name, (_, baseline_column, _) in REVIEW_SWEEPS.items():
frame = self.review_sweep_rows[name]
@@ -1782,10 +2557,10 @@ def review_judge_flagged_count(self) -> int:
@property
def engine_upgrade_rechecked_count(self) -> int:
- """Excluded outputs whose value moved on the new engine and were
- re-reviewed; they stay excluded."""
- revision = self.engine_upgrade_revision
- return 0 if revision is None else len(revision["excluded_outputs_rechecked"])
+ """Excluded outputs whose value moved on the last upgrade's engine and
+ were re-reviewed; they stay excluded."""
+ last = self.last_engine_upgrade
+ return 0 if last is None else last.rechecked_count
def _regenerated_keys_of_kind(self, kind: str) -> set[tuple[str, str]]:
return {
@@ -1877,6 +2652,180 @@ def later_law_exclusion_count(self) -> int:
1 for e in self.reference_exclusions if e["reason_code"] == LATER_LAW
)
+ @cached_property
+ def engine_defect_landed_keys(self) -> dict[str, frozenset[tuple[str, str]]]:
+ """Engine-defect records still excluded although the last upgrade's
+ engine computes the record's corrected value (within the exact-match
+ tolerance) or, where later engine changes moved the output, the
+ corrected value the audit's committed fix-module evidence gives on an
+ engine that still has the defect (upgrade_fix_evidence), so the
+ recorded defect is fixed there, split by why the
+ upgrade's recheck keeps each excluded: ``unlisted_input``, for the
+ second reason the record's note names (an input the prompt does not
+ state); ``further_defect``, because the recheck found the reference
+ rests on a further engine defect the new engine does not fix (its
+ reason says so). A landed record that is neither is refused."""
+ last = self.last_engine_upgrade
+ kinds: dict[str, set] = {"unlisted_input": set(), "further_defect": set()}
+ if last is None:
+ return {k: frozenset(v) for k, v in kinds.items()}
+ records = {
+ _key(entry): entry
+ for entry in self.reference_exclusions
+ if entry["reason_code"] == ENGINE_DEFECT
+ }
+ field = "value_on_" + last.engine_version.replace(".", "_")
+ for item in last.rechecked:
+ key = _key(item)
+ if key not in records or field not in item:
+ continue
+ record = records[key]
+ targets = [float(record["alternative_value"])]
+ targets += self.upgrade_fix_evidence.get(key, [])
+ if all(
+ moves_beyond_tolerance(item["variable"], target, float(item[field]))
+ for target in targets
+ ):
+ continue
+ if "(unlisted input)" in record.get("note", ""):
+ kinds["unlisted_input"].add(key)
+ elif FURTHER_DEFECT in item["reason"]:
+ kinds["further_defect"].add(key)
+ else:
+ raise ValueError(
+ f"{key} lands on its corrected value but its recheck names "
+ "neither an unstated input nor a further engine defect"
+ )
+ return {k: frozenset(v) for k, v in kinds.items()}
+
+ @cached_property
+ def upgrade_fix_evidence(self) -> dict[tuple[str, str], list[float]]:
+ """Per output, the corrected values the October audit's committed
+ fix-module evidence gives (reference_audit/2026-10-09-engine-upgrade/
+ evidence): each on an engine that still has the defect, kept only where
+ the modules moved the output beyond the exact-match tolerance there."""
+ found: dict[tuple[str, str], list[float]] = {}
+ for path in sorted((ROOT / OCTOBER_UPGRADE_AUDIT / "evidence").glob("*.json")):
+ doc = json.loads(path.read_text())
+ if doc.get("kind") != "regeneration_evidence":
+ continue
+ for item in doc["items"]:
+ if moves_beyond_tolerance(
+ item["variable"], item["engine_value"], item["corrected_value"]
+ ):
+ found.setdefault(_key(item), []).append(item["corrected_value"])
+ return found
+
+ @property
+ def engine_defect_fixed_kept_keys(self) -> frozenset[tuple[str, str]]:
+ """Engine-defect records whose defect the last upgrade's engine fixes
+ and which stay excluded for an input the prompt does not state."""
+ return self.engine_defect_landed_keys["unlisted_input"]
+
+ @property
+ def engine_defect_further_defect_keys(self) -> frozenset[tuple[str, str]]:
+ """Engine-defect records whose recorded defect the last upgrade's
+ engine fixes but whose reference rests on a further defect it does
+ not fix, so they stay excluded."""
+ return self.engine_defect_landed_keys["further_defect"]
+
+ @property
+ def engine_defect_fixed_kept_count(self) -> int:
+ return len(self.engine_defect_fixed_kept_keys)
+
+ @property
+ def engine_defect_further_defect_count(self) -> int:
+ return len(self.engine_defect_further_defect_keys)
+
+ @property
+ def engine_defect_fixed_kept_sentence(self) -> str:
+ """What the paper says of the fixed-but-kept records; empty for none."""
+ count = self.engine_defect_fixed_kept_count
+ if count == 0:
+ return ""
+ if count == 1:
+ return (
+ f"policyengine-us {self.policyengine_us_version} computes one of "
+ "them at its corrected value; it stays excluded because it also "
+ "moves under an input the prompt does not state."
+ )
+ return (
+ f"policyengine-us {self.policyengine_us_version} computes "
+ f"{count_word(count)} of them at their corrected values; they stay "
+ "excluded because each also moves under an input the prompt does not "
+ "state."
+ )
+
+ @property
+ def engine_defect_further_defect_sentence(self) -> str:
+ """What the paper says of the records kept for a further defect; empty
+ for none."""
+ count = self.engine_defect_further_defect_count
+ if count == 0:
+ return ""
+ version = self.policyengine_us_version
+ if count == 1:
+ return (
+ f"policyengine-us {version} computes one more at its recorded "
+ "corrected value, but its reference rests on a further engine "
+ f"defect, found on {version} and not fixed there, so it stays "
+ "excluded."
+ )
+ return (
+ f"policyengine-us {version} computes {count_word(count)} more at their "
+ "recorded corrected values, but each reference rests on a further "
+ f"engine defect, found on {version} and not fixed there, so they stay "
+ "excluded."
+ )
+
+ @property
+ def engine_defect_unfixed_count(self) -> int:
+ """Engine-defect exclusions whose recorded defect the current engine
+ has."""
+ return (
+ self.engine_defect_exclusion_count
+ - self.engine_defect_fixed_kept_count
+ - self.engine_defect_further_defect_count
+ )
+
+ @property
+ def engine_defect_present_count(self) -> int:
+ """Engine-defect exclusions whose reference rests on a defect the
+ current engine has: the recorded one, or a further one."""
+ return (
+ self.engine_defect_unfixed_count + self.engine_defect_further_defect_count
+ )
+
+ @property
+ def engine_defect_unfixed_root_cause_count(self) -> int:
+ """Distinct root causes behind the unfixed engine-defect exclusions."""
+ landed = (
+ self.engine_defect_fixed_kept_keys | self.engine_defect_further_defect_keys
+ )
+ causes: set[str] = set()
+ for entry in self.reference_exclusions:
+ if entry["reason_code"] == ENGINE_DEFECT and _key(entry) not in landed:
+ causes.update(exclusion_basis(entry).split("+"))
+ return len(causes)
+
+ @property
+ def engine_defect_present_root_cause_count(self) -> int:
+ """The unfixed root causes and the further defects, each a cause."""
+ last = self.last_engine_upgrade
+ reasons = {
+ item["reason"]
+ for item in ([] if last is None else last.rechecked)
+ if _key(item) in self.engine_defect_further_defect_keys
+ }
+ return self.engine_defect_unfixed_root_cause_count + len(reasons)
+
+ @property
+ def unstated_input_exclusion_total(self) -> int:
+ """Excluded outputs that depend on an input the prompt does not state:
+ the unlisted-input records, and the engine-defect records whose defect
+ is fixed and which stay excluded for such an input."""
+ return self.unlisted_input_exclusion_count + self.engine_defect_fixed_kept_count
+
@property
def engine_defect_root_cause_count(self) -> int:
"""Distinct root causes behind the engine-defect exclusions."""
diff --git a/policybench/reference_exclusions.py b/policybench/reference_exclusions.py
index 2906eb2b..7d0c0864 100644
--- a/policybench/reference_exclusions.py
+++ b/policybench/reference_exclusions.py
@@ -21,10 +21,11 @@
``reference_law_published_after_freeze``
The reference depends on a law or official parameter published after the
- references were frozen, so the frozen value is the engine's projection
- and no model could have known the governing figure when it answered. The
- entry names what was published and when, and records the frozen
- projection beside the value under the published figure.
+ references were frozen, so the frozen value is the engine's own figure (a
+ projection, or a computation no government stated before the freeze) and
+ no model could have known the governing figure when it answered. The
+ entry names what was published and when, and records the frozen value
+ beside the value under the published figure.
The record lives beside the reference CSV as ``reference_exclusions.json`` and
travels with the run into the frozen snapshot, where the manifest pins it.
diff --git a/pyproject.toml b/pyproject.toml
index d40404bc..8c564897 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
"pandas>=2.0",
"numpy>=1.24",
"policyengine==6.1.2",
- "policyengine-us==2.15.17",
+ "policyengine-us==2.38.6",
"policyengine-uk==2.89.0",
"anthropic>=0.115.1",
"google-genai>=2.10.0",
diff --git a/reference_audit/2026-09-28/fixes/r19_irs_sales_tax_2025.json b/reference_audit/2026-09-28/fixes/r19_irs_sales_tax_2025.json
new file mode 100644
index 00000000..f327044e
--- /dev/null
+++ b/reference_audit/2026-09-28/fixes/r19_irs_sales_tax_2025.json
@@ -0,0 +1,6530 @@
+{
+ "AK": [
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ ],
+ "AL": [
+ [
+ 306,
+ 405,
+ 453,
+ 492,
+ 525,
+ 555,
+ 581,
+ 605,
+ 628,
+ 657,
+ 695,
+ 728,
+ 759,
+ 787,
+ 816,
+ 846,
+ 874,
+ 902,
+ 1052
+ ],
+ [
+ 371,
+ 491,
+ 547,
+ 594,
+ 635,
+ 670,
+ 702,
+ 731,
+ 758,
+ 793,
+ 838,
+ 877,
+ 914,
+ 949,
+ 983,
+ 1020,
+ 1053,
+ 1086,
+ 1265
+ ],
+ [
+ 416,
+ 550,
+ 613,
+ 666,
+ 711,
+ 751,
+ 787,
+ 820,
+ 849,
+ 889,
+ 938,
+ 983,
+ 1024,
+ 1061,
+ 1100,
+ 1142,
+ 1179,
+ 1215,
+ 1416
+ ],
+ [
+ 453,
+ 597,
+ 666,
+ 723,
+ 773,
+ 816,
+ 854,
+ 890,
+ 922,
+ 964,
+ 1018,
+ 1066,
+ 1111,
+ 1152,
+ 1193,
+ 1238,
+ 1279,
+ 1317,
+ 1535
+ ],
+ [
+ 483,
+ 638,
+ 712,
+ 772,
+ 825,
+ 870,
+ 911,
+ 949,
+ 983,
+ 1029,
+ 1086,
+ 1137,
+ 1184,
+ 1228,
+ 1273,
+ 1319,
+ 1363,
+ 1405,
+ 1635
+ ],
+ [
+ 527,
+ 696,
+ 776,
+ 842,
+ 899,
+ 949,
+ 993,
+ 1034,
+ 1071,
+ 1120,
+ 1182,
+ 1239,
+ 1290,
+ 1338,
+ 1385,
+ 1437,
+ 1484,
+ 1528,
+ 1779
+ ]
+ ],
+ "AR": [
+ [
+ 420,
+ 584,
+ 665,
+ 733,
+ 793,
+ 846,
+ 894,
+ 937,
+ 978,
+ 1032,
+ 1100,
+ 1163,
+ 1221,
+ 1274,
+ 1329,
+ 1387,
+ 1442,
+ 1494,
+ 1790
+ ],
+ [
+ 486,
+ 676,
+ 770,
+ 849,
+ 917,
+ 978,
+ 1033,
+ 1084,
+ 1131,
+ 1193,
+ 1273,
+ 1345,
+ 1412,
+ 1473,
+ 1537,
+ 1605,
+ 1668,
+ 1726,
+ 2070
+ ],
+ [
+ 530,
+ 737,
+ 840,
+ 926,
+ 1000,
+ 1067,
+ 1127,
+ 1183,
+ 1234,
+ 1302,
+ 1388,
+ 1467,
+ 1540,
+ 1607,
+ 1677,
+ 1750,
+ 1819,
+ 1884,
+ 2258
+ ],
+ [
+ 565,
+ 786,
+ 895,
+ 986,
+ 1065,
+ 1136,
+ 1200,
+ 1259,
+ 1314,
+ 1386,
+ 1478,
+ 1562,
+ 1639,
+ 1711,
+ 1785,
+ 1864,
+ 1937,
+ 2006,
+ 2403
+ ],
+ [
+ 593,
+ 826,
+ 939,
+ 1036,
+ 1119,
+ 1193,
+ 1261,
+ 1323,
+ 1380,
+ 1455,
+ 1553,
+ 1641,
+ 1721,
+ 1798,
+ 1875,
+ 1957,
+ 2034,
+ 2106,
+ 2524
+ ],
+ [
+ 634,
+ 882,
+ 1003,
+ 1106,
+ 1195,
+ 1274,
+ 1346,
+ 1412,
+ 1474,
+ 1554,
+ 1657,
+ 1752,
+ 1837,
+ 1918,
+ 2001,
+ 2089,
+ 2170,
+ 2248,
+ 2693
+ ]
+ ],
+ "AZ": [
+ [
+ 335,
+ 463,
+ 525,
+ 578,
+ 624,
+ 664,
+ 701,
+ 734,
+ 766,
+ 806,
+ 859,
+ 907,
+ 951,
+ 991,
+ 1034,
+ 1078,
+ 1119,
+ 1159,
+ 1383
+ ],
+ [
+ 386,
+ 532,
+ 605,
+ 665,
+ 718,
+ 765,
+ 806,
+ 845,
+ 882,
+ 928,
+ 989,
+ 1044,
+ 1095,
+ 1142,
+ 1190,
+ 1241,
+ 1289,
+ 1334,
+ 1592
+ ],
+ [
+ 419,
+ 579,
+ 658,
+ 724,
+ 781,
+ 832,
+ 877,
+ 920,
+ 959,
+ 1010,
+ 1077,
+ 1136,
+ 1191,
+ 1243,
+ 1296,
+ 1351,
+ 1404,
+ 1452,
+ 1734
+ ],
+ [
+ 446,
+ 615,
+ 700,
+ 769,
+ 830,
+ 885,
+ 933,
+ 978,
+ 1020,
+ 1074,
+ 1145,
+ 1209,
+ 1266,
+ 1321,
+ 1377,
+ 1436,
+ 1492,
+ 1544,
+ 1843
+ ],
+ [
+ 467,
+ 646,
+ 733,
+ 806,
+ 870,
+ 927,
+ 979,
+ 1026,
+ 1069,
+ 1127,
+ 1200,
+ 1267,
+ 1328,
+ 1385,
+ 1444,
+ 1507,
+ 1565,
+ 1620,
+ 1934
+ ],
+ [
+ 498,
+ 688,
+ 781,
+ 859,
+ 927,
+ 988,
+ 1043,
+ 1093,
+ 1139,
+ 1200,
+ 1279,
+ 1351,
+ 1416,
+ 1477,
+ 1539,
+ 1606,
+ 1668,
+ 1725,
+ 2061
+ ]
+ ],
+ "CA": [
+ [
+ 412,
+ 564,
+ 637,
+ 698,
+ 752,
+ 798,
+ 841,
+ 879,
+ 915,
+ 963,
+ 1023,
+ 1079,
+ 1128,
+ 1175,
+ 1223,
+ 1274,
+ 1320,
+ 1365,
+ 1619
+ ],
+ [
+ 476,
+ 650,
+ 734,
+ 805,
+ 866,
+ 921,
+ 970,
+ 1015,
+ 1056,
+ 1110,
+ 1180,
+ 1243,
+ 1301,
+ 1355,
+ 1410,
+ 1469,
+ 1522,
+ 1574,
+ 1867
+ ],
+ [
+ 519,
+ 709,
+ 801,
+ 877,
+ 944,
+ 1003,
+ 1057,
+ 1106,
+ 1151,
+ 1211,
+ 1286,
+ 1355,
+ 1418,
+ 1477,
+ 1537,
+ 1600,
+ 1659,
+ 1715,
+ 2034
+ ],
+ [
+ 552,
+ 754,
+ 852,
+ 934,
+ 1005,
+ 1067,
+ 1124,
+ 1176,
+ 1225,
+ 1288,
+ 1368,
+ 1442,
+ 1508,
+ 1571,
+ 1635,
+ 1702,
+ 1765,
+ 1825,
+ 2164
+ ],
+ [
+ 580,
+ 792,
+ 895,
+ 981,
+ 1055,
+ 1121,
+ 1181,
+ 1235,
+ 1286,
+ 1352,
+ 1437,
+ 1514,
+ 1584,
+ 1649,
+ 1716,
+ 1787,
+ 1853,
+ 1915,
+ 2272
+ ],
+ [
+ 619,
+ 845,
+ 955,
+ 1046,
+ 1126,
+ 1196,
+ 1259,
+ 1318,
+ 1372,
+ 1442,
+ 1532,
+ 1615,
+ 1690,
+ 1760,
+ 1831,
+ 1907,
+ 1977,
+ 2043,
+ 2423
+ ]
+ ],
+ "CO": [
+ [
+ 170,
+ 233,
+ 263,
+ 289,
+ 312,
+ 331,
+ 349,
+ 366,
+ 381,
+ 400,
+ 427,
+ 449,
+ 470,
+ 491,
+ 511,
+ 532,
+ 551,
+ 571,
+ 678
+ ],
+ [
+ 197,
+ 271,
+ 307,
+ 336,
+ 363,
+ 386,
+ 406,
+ 426,
+ 443,
+ 466,
+ 496,
+ 523,
+ 547,
+ 571,
+ 594,
+ 619,
+ 642,
+ 664,
+ 790
+ ],
+ [
+ 216,
+ 297,
+ 336,
+ 369,
+ 397,
+ 422,
+ 445,
+ 466,
+ 485,
+ 511,
+ 543,
+ 573,
+ 600,
+ 626,
+ 651,
+ 678,
+ 704,
+ 728,
+ 865
+ ],
+ [
+ 232,
+ 317,
+ 358,
+ 394,
+ 425,
+ 451,
+ 475,
+ 498,
+ 519,
+ 545,
+ 581,
+ 612,
+ 641,
+ 668,
+ 696,
+ 725,
+ 752,
+ 778,
+ 924
+ ],
+ [
+ 244,
+ 334,
+ 378,
+ 414,
+ 447,
+ 475,
+ 501,
+ 524,
+ 546,
+ 575,
+ 611,
+ 645,
+ 675,
+ 704,
+ 732,
+ 763,
+ 792,
+ 819,
+ 974
+ ],
+ [
+ 261,
+ 357,
+ 405,
+ 445,
+ 478,
+ 509,
+ 536,
+ 562,
+ 585,
+ 615,
+ 655,
+ 691,
+ 723,
+ 754,
+ 785,
+ 818,
+ 848,
+ 877,
+ 1043
+ ]
+ ],
+ "CT": [
+ [
+ 378,
+ 519,
+ 589,
+ 647,
+ 698,
+ 742,
+ 782,
+ 820,
+ 854,
+ 899,
+ 957,
+ 1009,
+ 1057,
+ 1103,
+ 1149,
+ 1197,
+ 1243,
+ 1286,
+ 1531
+ ],
+ [
+ 416,
+ 572,
+ 649,
+ 712,
+ 768,
+ 817,
+ 861,
+ 902,
+ 940,
+ 990,
+ 1053,
+ 1111,
+ 1164,
+ 1214,
+ 1264,
+ 1317,
+ 1368,
+ 1415,
+ 1685
+ ],
+ [
+ 441,
+ 605,
+ 687,
+ 754,
+ 812,
+ 865,
+ 912,
+ 955,
+ 995,
+ 1048,
+ 1115,
+ 1176,
+ 1232,
+ 1285,
+ 1338,
+ 1394,
+ 1447,
+ 1497,
+ 1783
+ ],
+ [
+ 459,
+ 631,
+ 715,
+ 785,
+ 847,
+ 901,
+ 950,
+ 994,
+ 1036,
+ 1091,
+ 1161,
+ 1225,
+ 1283,
+ 1338,
+ 1393,
+ 1452,
+ 1507,
+ 1559,
+ 1856
+ ],
+ [
+ 474,
+ 651,
+ 738,
+ 810,
+ 874,
+ 930,
+ 980,
+ 1027,
+ 1069,
+ 1126,
+ 1198,
+ 1264,
+ 1324,
+ 1380,
+ 1438,
+ 1499,
+ 1556,
+ 1609,
+ 1916
+ ],
+ [
+ 495,
+ 679,
+ 770,
+ 846,
+ 911,
+ 970,
+ 1023,
+ 1070,
+ 1115,
+ 1174,
+ 1249,
+ 1318,
+ 1381,
+ 1439,
+ 1499,
+ 1563,
+ 1622,
+ 1678,
+ 1998
+ ]
+ ],
+ "DC": [
+ [
+ 363,
+ 493,
+ 557,
+ 609,
+ 655,
+ 695,
+ 731,
+ 765,
+ 796,
+ 837,
+ 889,
+ 935,
+ 978,
+ 1019,
+ 1059,
+ 1103,
+ 1143,
+ 1181,
+ 1398
+ ],
+ [
+ 398,
+ 541,
+ 611,
+ 669,
+ 719,
+ 764,
+ 804,
+ 841,
+ 874,
+ 919,
+ 976,
+ 1028,
+ 1074,
+ 1119,
+ 1164,
+ 1212,
+ 1255,
+ 1297,
+ 1536
+ ],
+ [
+ 421,
+ 573,
+ 647,
+ 708,
+ 761,
+ 807,
+ 850,
+ 889,
+ 925,
+ 972,
+ 1032,
+ 1087,
+ 1136,
+ 1183,
+ 1231,
+ 1281,
+ 1327,
+ 1371,
+ 1623
+ ],
+ [
+ 439,
+ 596,
+ 673,
+ 736,
+ 792,
+ 841,
+ 885,
+ 925,
+ 963,
+ 1012,
+ 1074,
+ 1131,
+ 1183,
+ 1231,
+ 1281,
+ 1332,
+ 1381,
+ 1427,
+ 1689
+ ],
+ [
+ 453,
+ 615,
+ 695,
+ 760,
+ 818,
+ 867,
+ 913,
+ 955,
+ 993,
+ 1044,
+ 1108,
+ 1167,
+ 1221,
+ 1269,
+ 1321,
+ 1375,
+ 1425,
+ 1473,
+ 1743
+ ],
+ [
+ 472,
+ 641,
+ 724,
+ 792,
+ 852,
+ 904,
+ 952,
+ 994,
+ 1035,
+ 1088,
+ 1155,
+ 1216,
+ 1271,
+ 1323,
+ 1376,
+ 1432,
+ 1485,
+ 1534,
+ 1815
+ ]
+ ],
+ "DE": [
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ ],
+ "FL": [
+ [
+ 379,
+ 524,
+ 596,
+ 656,
+ 709,
+ 755,
+ 797,
+ 836,
+ 871,
+ 918,
+ 979,
+ 1034,
+ 1084,
+ 1131,
+ 1179,
+ 1230,
+ 1278,
+ 1323,
+ 1582
+ ],
+ [
+ 433,
+ 599,
+ 681,
+ 751,
+ 810,
+ 863,
+ 911,
+ 956,
+ 996,
+ 1050,
+ 1119,
+ 1182,
+ 1240,
+ 1294,
+ 1349,
+ 1408,
+ 1461,
+ 1513,
+ 1810
+ ],
+ [
+ 468,
+ 649,
+ 738,
+ 812,
+ 877,
+ 935,
+ 987,
+ 1035,
+ 1080,
+ 1137,
+ 1213,
+ 1281,
+ 1344,
+ 1401,
+ 1461,
+ 1525,
+ 1584,
+ 1640,
+ 1961
+ ],
+ [
+ 497,
+ 688,
+ 782,
+ 861,
+ 929,
+ 990,
+ 1046,
+ 1097,
+ 1144,
+ 1205,
+ 1285,
+ 1357,
+ 1423,
+ 1485,
+ 1549,
+ 1616,
+ 1679,
+ 1738,
+ 2078
+ ],
+ [
+ 519,
+ 719,
+ 818,
+ 901,
+ 972,
+ 1036,
+ 1094,
+ 1148,
+ 1196,
+ 1261,
+ 1344,
+ 1420,
+ 1489,
+ 1554,
+ 1620,
+ 1691,
+ 1756,
+ 1818,
+ 2174
+ ],
+ [
+ 550,
+ 764,
+ 868,
+ 956,
+ 1032,
+ 1100,
+ 1162,
+ 1218,
+ 1270,
+ 1339,
+ 1427,
+ 1507,
+ 1581,
+ 1649,
+ 1720,
+ 1795,
+ 1865,
+ 1931,
+ 2309
+ ]
+ ],
+ "GA": [
+ [
+ 235,
+ 324,
+ 368,
+ 404,
+ 436,
+ 464,
+ 490,
+ 513,
+ 535,
+ 564,
+ 600,
+ 633,
+ 663,
+ 692,
+ 721,
+ 752,
+ 781,
+ 808,
+ 964
+ ],
+ [
+ 270,
+ 373,
+ 423,
+ 466,
+ 503,
+ 535,
+ 565,
+ 591,
+ 616,
+ 649,
+ 692,
+ 730,
+ 765,
+ 798,
+ 832,
+ 867,
+ 901,
+ 931,
+ 1112
+ ],
+ [
+ 295,
+ 406,
+ 461,
+ 507,
+ 547,
+ 583,
+ 614,
+ 644,
+ 671,
+ 708,
+ 754,
+ 795,
+ 834,
+ 869,
+ 906,
+ 944,
+ 981,
+ 1015,
+ 1212
+ ],
+ [
+ 313,
+ 432,
+ 491,
+ 539,
+ 582,
+ 619,
+ 654,
+ 686,
+ 714,
+ 753,
+ 801,
+ 846,
+ 887,
+ 924,
+ 964,
+ 1004,
+ 1043,
+ 1080,
+ 1289
+ ],
+ [
+ 328,
+ 453,
+ 515,
+ 566,
+ 610,
+ 650,
+ 687,
+ 719,
+ 749,
+ 790,
+ 841,
+ 888,
+ 930,
+ 970,
+ 1012,
+ 1054,
+ 1095,
+ 1133,
+ 1353
+ ],
+ [
+ 350,
+ 483,
+ 548,
+ 603,
+ 651,
+ 694,
+ 731,
+ 767,
+ 799,
+ 842,
+ 897,
+ 947,
+ 992,
+ 1035,
+ 1079,
+ 1124,
+ 1168,
+ 1209,
+ 1442
+ ]
+ ],
+ "HI": [
+ [
+ 397,
+ 537,
+ 604,
+ 660,
+ 709,
+ 752,
+ 790,
+ 826,
+ 858,
+ 901,
+ 956,
+ 1005,
+ 1050,
+ 1093,
+ 1135,
+ 1181,
+ 1224,
+ 1263,
+ 1491
+ ],
+ [
+ 475,
+ 641,
+ 721,
+ 788,
+ 845,
+ 897,
+ 942,
+ 984,
+ 1023,
+ 1073,
+ 1138,
+ 1197,
+ 1251,
+ 1301,
+ 1352,
+ 1406,
+ 1456,
+ 1503,
+ 1772
+ ],
+ [
+ 528,
+ 712,
+ 801,
+ 875,
+ 939,
+ 995,
+ 1046,
+ 1093,
+ 1135,
+ 1192,
+ 1263,
+ 1329,
+ 1388,
+ 1443,
+ 1500,
+ 1560,
+ 1615,
+ 1668,
+ 1965
+ ],
+ [
+ 570,
+ 769,
+ 864,
+ 943,
+ 1013,
+ 1073,
+ 1128,
+ 1178,
+ 1225,
+ 1285,
+ 1362,
+ 1433,
+ 1496,
+ 1556,
+ 1617,
+ 1681,
+ 1741,
+ 1797,
+ 2117
+ ],
+ [
+ 605,
+ 816,
+ 917,
+ 1001,
+ 1074,
+ 1138,
+ 1196,
+ 1249,
+ 1298,
+ 1363,
+ 1444,
+ 1519,
+ 1586,
+ 1649,
+ 1713,
+ 1781,
+ 1845,
+ 1904,
+ 2244
+ ],
+ [
+ 654,
+ 882,
+ 991,
+ 1083,
+ 1161,
+ 1231,
+ 1293,
+ 1351,
+ 1404,
+ 1473,
+ 1561,
+ 1641,
+ 1714,
+ 1782,
+ 1851,
+ 1925,
+ 1993,
+ 2058,
+ 2423
+ ]
+ ],
+ "IA": [
+ [
+ 401,
+ 551,
+ 626,
+ 687,
+ 740,
+ 787,
+ 830,
+ 869,
+ 906,
+ 954,
+ 1015,
+ 1070,
+ 1121,
+ 1169,
+ 1218,
+ 1269,
+ 1317,
+ 1363,
+ 1623
+ ],
+ [
+ 464,
+ 638,
+ 723,
+ 794,
+ 856,
+ 910,
+ 960,
+ 1005,
+ 1047,
+ 1102,
+ 1173,
+ 1237,
+ 1296,
+ 1351,
+ 1407,
+ 1466,
+ 1521,
+ 1574,
+ 1874
+ ],
+ [
+ 507,
+ 696,
+ 789,
+ 866,
+ 933,
+ 993,
+ 1047,
+ 1096,
+ 1142,
+ 1201,
+ 1279,
+ 1349,
+ 1413,
+ 1473,
+ 1534,
+ 1599,
+ 1658,
+ 1715,
+ 2041
+ ],
+ [
+ 539,
+ 741,
+ 840,
+ 922,
+ 993,
+ 1056,
+ 1114,
+ 1167,
+ 1215,
+ 1279,
+ 1361,
+ 1435,
+ 1503,
+ 1567,
+ 1632,
+ 1700,
+ 1765,
+ 1825,
+ 2172
+ ],
+ [
+ 567,
+ 778,
+ 882,
+ 968,
+ 1043,
+ 1109,
+ 1170,
+ 1225,
+ 1276,
+ 1343,
+ 1429,
+ 1507,
+ 1578,
+ 1644,
+ 1713,
+ 1785,
+ 1852,
+ 1916,
+ 2280
+ ],
+ [
+ 605,
+ 831,
+ 940,
+ 1033,
+ 1113,
+ 1183,
+ 1247,
+ 1306,
+ 1361,
+ 1432,
+ 1523,
+ 1608,
+ 1683,
+ 1754,
+ 1827,
+ 1903,
+ 1975,
+ 2043,
+ 2431
+ ]
+ ],
+ "ID": [
+ [
+ 504,
+ 671,
+ 752,
+ 818,
+ 875,
+ 926,
+ 971,
+ 1013,
+ 1051,
+ 1101,
+ 1165,
+ 1223,
+ 1276,
+ 1324,
+ 1374,
+ 1426,
+ 1475,
+ 1521,
+ 1782
+ ],
+ [
+ 611,
+ 813,
+ 911,
+ 991,
+ 1060,
+ 1121,
+ 1176,
+ 1226,
+ 1273,
+ 1332,
+ 1410,
+ 1480,
+ 1543,
+ 1601,
+ 1661,
+ 1724,
+ 1783,
+ 1838,
+ 2152
+ ],
+ [
+ 688,
+ 914,
+ 1023,
+ 1113,
+ 1190,
+ 1258,
+ 1320,
+ 1376,
+ 1428,
+ 1495,
+ 1581,
+ 1658,
+ 1730,
+ 1796,
+ 1863,
+ 1933,
+ 1999,
+ 2061,
+ 2411
+ ],
+ [
+ 747,
+ 994,
+ 1112,
+ 1210,
+ 1294,
+ 1368,
+ 1434,
+ 1495,
+ 1551,
+ 1624,
+ 1717,
+ 1802,
+ 1879,
+ 1950,
+ 2022,
+ 2099,
+ 2170,
+ 2237,
+ 2617
+ ],
+ [
+ 798,
+ 1061,
+ 1187,
+ 1292,
+ 1381,
+ 1460,
+ 1530,
+ 1595,
+ 1655,
+ 1734,
+ 1832,
+ 1922,
+ 2005,
+ 2080,
+ 2157,
+ 2239,
+ 2315,
+ 2387,
+ 2791
+ ],
+ [
+ 871,
+ 1158,
+ 1295,
+ 1409,
+ 1506,
+ 1591,
+ 1669,
+ 1740,
+ 1805,
+ 1889,
+ 1998,
+ 2095,
+ 2184,
+ 2267,
+ 2351,
+ 2439,
+ 2523,
+ 2600,
+ 3040
+ ]
+ ],
+ "IL": [
+ [
+ 356,
+ 485,
+ 548,
+ 600,
+ 646,
+ 686,
+ 722,
+ 755,
+ 785,
+ 826,
+ 876,
+ 924,
+ 966,
+ 1005,
+ 1047,
+ 1090,
+ 1129,
+ 1168,
+ 1383
+ ],
+ [
+ 421,
+ 573,
+ 646,
+ 707,
+ 760,
+ 806,
+ 849,
+ 888,
+ 923,
+ 971,
+ 1031,
+ 1086,
+ 1135,
+ 1181,
+ 1229,
+ 1279,
+ 1325,
+ 1370,
+ 1621
+ ],
+ [
+ 466,
+ 633,
+ 714,
+ 781,
+ 840,
+ 892,
+ 938,
+ 981,
+ 1020,
+ 1071,
+ 1137,
+ 1198,
+ 1252,
+ 1303,
+ 1356,
+ 1411,
+ 1462,
+ 1510,
+ 1787
+ ],
+ [
+ 503,
+ 681,
+ 769,
+ 841,
+ 903,
+ 959,
+ 1008,
+ 1054,
+ 1097,
+ 1152,
+ 1223,
+ 1288,
+ 1346,
+ 1400,
+ 1456,
+ 1515,
+ 1570,
+ 1623,
+ 1918
+ ],
+ [
+ 532,
+ 722,
+ 813,
+ 891,
+ 957,
+ 1015,
+ 1067,
+ 1116,
+ 1161,
+ 1220,
+ 1294,
+ 1363,
+ 1425,
+ 1482,
+ 1542,
+ 1604,
+ 1661,
+ 1716,
+ 2030
+ ],
+ [
+ 576,
+ 780,
+ 878,
+ 961,
+ 1033,
+ 1096,
+ 1153,
+ 1204,
+ 1253,
+ 1316,
+ 1396,
+ 1470,
+ 1537,
+ 1599,
+ 1662,
+ 1730,
+ 1792,
+ 1851,
+ 2188
+ ]
+ ],
+ "IN": [
+ [
+ 408,
+ 564,
+ 640,
+ 704,
+ 759,
+ 807,
+ 852,
+ 893,
+ 930,
+ 980,
+ 1044,
+ 1102,
+ 1155,
+ 1204,
+ 1255,
+ 1308,
+ 1359,
+ 1406,
+ 1677
+ ],
+ [
+ 475,
+ 655,
+ 743,
+ 817,
+ 880,
+ 937,
+ 989,
+ 1037,
+ 1081,
+ 1137,
+ 1212,
+ 1279,
+ 1340,
+ 1397,
+ 1455,
+ 1518,
+ 1576,
+ 1631,
+ 1945
+ ],
+ [
+ 520,
+ 716,
+ 812,
+ 894,
+ 964,
+ 1026,
+ 1082,
+ 1133,
+ 1181,
+ 1244,
+ 1324,
+ 1398,
+ 1465,
+ 1527,
+ 1591,
+ 1659,
+ 1723,
+ 1782,
+ 2126
+ ],
+ [
+ 555,
+ 764,
+ 867,
+ 953,
+ 1028,
+ 1094,
+ 1154,
+ 1210,
+ 1260,
+ 1327,
+ 1413,
+ 1491,
+ 1563,
+ 1629,
+ 1697,
+ 1770,
+ 1837,
+ 1901,
+ 2267
+ ],
+ [
+ 584,
+ 804,
+ 912,
+ 1002,
+ 1082,
+ 1151,
+ 1214,
+ 1271,
+ 1325,
+ 1395,
+ 1486,
+ 1568,
+ 1643,
+ 1713,
+ 1785,
+ 1862,
+ 1933,
+ 2000,
+ 2385
+ ],
+ [
+ 625,
+ 860,
+ 976,
+ 1072,
+ 1157,
+ 1231,
+ 1298,
+ 1360,
+ 1418,
+ 1493,
+ 1589,
+ 1677,
+ 1757,
+ 1832,
+ 1909,
+ 1991,
+ 2067,
+ 2138,
+ 2549
+ ]
+ ],
+ "KS": [
+ [
+ 417,
+ 572,
+ 647,
+ 710,
+ 765,
+ 813,
+ 857,
+ 898,
+ 934,
+ 984,
+ 1046,
+ 1103,
+ 1155,
+ 1203,
+ 1253,
+ 1306,
+ 1355,
+ 1400,
+ 1666
+ ],
+ [
+ 493,
+ 675,
+ 764,
+ 838,
+ 903,
+ 960,
+ 1010,
+ 1058,
+ 1102,
+ 1160,
+ 1233,
+ 1300,
+ 1361,
+ 1418,
+ 1477,
+ 1538,
+ 1595,
+ 1649,
+ 1961
+ ],
+ [
+ 544,
+ 745,
+ 844,
+ 925,
+ 996,
+ 1059,
+ 1116,
+ 1169,
+ 1217,
+ 1281,
+ 1361,
+ 1435,
+ 1502,
+ 1565,
+ 1629,
+ 1697,
+ 1761,
+ 1821,
+ 2163
+ ],
+ [
+ 586,
+ 801,
+ 907,
+ 994,
+ 1070,
+ 1138,
+ 1199,
+ 1255,
+ 1307,
+ 1375,
+ 1461,
+ 1542,
+ 1614,
+ 1681,
+ 1750,
+ 1823,
+ 1891,
+ 1955,
+ 2322
+ ],
+ [
+ 619,
+ 848,
+ 960,
+ 1052,
+ 1132,
+ 1204,
+ 1268,
+ 1327,
+ 1382,
+ 1454,
+ 1546,
+ 1630,
+ 1706,
+ 1777,
+ 1850,
+ 1928,
+ 1999,
+ 2067,
+ 2455
+ ],
+ [
+ 668,
+ 914,
+ 1034,
+ 1133,
+ 1221,
+ 1297,
+ 1367,
+ 1430,
+ 1489,
+ 1567,
+ 1666,
+ 1756,
+ 1837,
+ 1914,
+ 1993,
+ 2076,
+ 2153,
+ 2226,
+ 2644
+ ]
+ ],
+ "KY": [
+ [
+ 388,
+ 537,
+ 610,
+ 672,
+ 726,
+ 773,
+ 817,
+ 856,
+ 893,
+ 940,
+ 1002,
+ 1059,
+ 1110,
+ 1159,
+ 1208,
+ 1260,
+ 1309,
+ 1355,
+ 1620
+ ],
+ [
+ 443,
+ 613,
+ 698,
+ 768,
+ 829,
+ 884,
+ 932,
+ 978,
+ 1020,
+ 1074,
+ 1146,
+ 1211,
+ 1268,
+ 1324,
+ 1380,
+ 1440,
+ 1496,
+ 1549,
+ 1852
+ ],
+ [
+ 479,
+ 664,
+ 755,
+ 831,
+ 898,
+ 957,
+ 1009,
+ 1058,
+ 1104,
+ 1164,
+ 1240,
+ 1310,
+ 1374,
+ 1434,
+ 1495,
+ 1560,
+ 1621,
+ 1678,
+ 2007
+ ],
+ [
+ 507,
+ 703,
+ 799,
+ 879,
+ 951,
+ 1013,
+ 1069,
+ 1121,
+ 1169,
+ 1232,
+ 1313,
+ 1387,
+ 1455,
+ 1518,
+ 1583,
+ 1652,
+ 1716,
+ 1777,
+ 2126
+ ],
+ [
+ 530,
+ 734,
+ 836,
+ 920,
+ 993,
+ 1059,
+ 1118,
+ 1172,
+ 1223,
+ 1289,
+ 1374,
+ 1451,
+ 1522,
+ 1588,
+ 1655,
+ 1727,
+ 1795,
+ 1858,
+ 2223
+ ],
+ [
+ 563,
+ 779,
+ 887,
+ 976,
+ 1054,
+ 1123,
+ 1186,
+ 1244,
+ 1297,
+ 1367,
+ 1457,
+ 1540,
+ 1615,
+ 1685,
+ 1757,
+ 1833,
+ 1904,
+ 1972,
+ 2359
+ ]
+ ],
+ "LA": [
+ [
+ 313,
+ 434,
+ 493,
+ 543,
+ 587,
+ 625,
+ 661,
+ 693,
+ 722,
+ 762,
+ 812,
+ 858,
+ 900,
+ 939,
+ 980,
+ 1022,
+ 1062,
+ 1100,
+ 1317
+ ],
+ [
+ 356,
+ 494,
+ 563,
+ 620,
+ 670,
+ 714,
+ 754,
+ 791,
+ 825,
+ 871,
+ 928,
+ 980,
+ 1028,
+ 1074,
+ 1119,
+ 1168,
+ 1214,
+ 1257,
+ 1505
+ ],
+ [
+ 386,
+ 535,
+ 609,
+ 671,
+ 726,
+ 774,
+ 817,
+ 857,
+ 893,
+ 943,
+ 1005,
+ 1062,
+ 1114,
+ 1163,
+ 1213,
+ 1265,
+ 1315,
+ 1361,
+ 1631
+ ],
+ [
+ 409,
+ 567,
+ 645,
+ 711,
+ 768,
+ 818,
+ 865,
+ 907,
+ 946,
+ 997,
+ 1063,
+ 1124,
+ 1180,
+ 1231,
+ 1284,
+ 1341,
+ 1393,
+ 1442,
+ 1728
+ ],
+ [
+ 427,
+ 592,
+ 674,
+ 743,
+ 803,
+ 856,
+ 905,
+ 948,
+ 989,
+ 1044,
+ 1113,
+ 1176,
+ 1234,
+ 1287,
+ 1343,
+ 1402,
+ 1457,
+ 1509,
+ 1806
+ ],
+ [
+ 453,
+ 629,
+ 715,
+ 788,
+ 852,
+ 908,
+ 960,
+ 1006,
+ 1050,
+ 1107,
+ 1181,
+ 1248,
+ 1309,
+ 1367,
+ 1425,
+ 1488,
+ 1546,
+ 1601,
+ 1918
+ ]
+ ],
+ "MA": [
+ [
+ 318,
+ 432,
+ 486,
+ 532,
+ 572,
+ 607,
+ 639,
+ 668,
+ 695,
+ 730,
+ 775,
+ 816,
+ 853,
+ 888,
+ 923,
+ 961,
+ 995,
+ 1028,
+ 1216
+ ],
+ [
+ 362,
+ 491,
+ 552,
+ 605,
+ 650,
+ 690,
+ 726,
+ 759,
+ 789,
+ 829,
+ 879,
+ 926,
+ 969,
+ 1007,
+ 1048,
+ 1091,
+ 1130,
+ 1168,
+ 1380
+ ],
+ [
+ 390,
+ 529,
+ 597,
+ 653,
+ 702,
+ 744,
+ 784,
+ 820,
+ 852,
+ 896,
+ 951,
+ 1000,
+ 1046,
+ 1089,
+ 1131,
+ 1178,
+ 1221,
+ 1260,
+ 1491
+ ],
+ [
+ 412,
+ 560,
+ 631,
+ 691,
+ 742,
+ 787,
+ 829,
+ 866,
+ 901,
+ 947,
+ 1004,
+ 1057,
+ 1106,
+ 1151,
+ 1196,
+ 1245,
+ 1290,
+ 1332,
+ 1575
+ ],
+ [
+ 431,
+ 585,
+ 659,
+ 721,
+ 775,
+ 823,
+ 865,
+ 905,
+ 941,
+ 988,
+ 1049,
+ 1105,
+ 1155,
+ 1201,
+ 1249,
+ 1300,
+ 1348,
+ 1391,
+ 1645
+ ],
+ [
+ 457,
+ 619,
+ 699,
+ 765,
+ 822,
+ 871,
+ 917,
+ 959,
+ 997,
+ 1047,
+ 1112,
+ 1170,
+ 1224,
+ 1273,
+ 1324,
+ 1377,
+ 1428,
+ 1475,
+ 1743
+ ]
+ ],
+ "MD": [
+ [
+ 301,
+ 413,
+ 468,
+ 515,
+ 555,
+ 590,
+ 623,
+ 652,
+ 679,
+ 715,
+ 761,
+ 803,
+ 841,
+ 877,
+ 914,
+ 953,
+ 988,
+ 1023,
+ 1219
+ ],
+ [
+ 350,
+ 481,
+ 545,
+ 599,
+ 646,
+ 687,
+ 724,
+ 759,
+ 790,
+ 832,
+ 885,
+ 933,
+ 977,
+ 1019,
+ 1061,
+ 1106,
+ 1148,
+ 1187,
+ 1414
+ ],
+ [
+ 386,
+ 529,
+ 599,
+ 658,
+ 709,
+ 754,
+ 794,
+ 832,
+ 866,
+ 912,
+ 970,
+ 1024,
+ 1071,
+ 1117,
+ 1164,
+ 1213,
+ 1258,
+ 1301,
+ 1549
+ ],
+ [
+ 413,
+ 567,
+ 642,
+ 704,
+ 759,
+ 806,
+ 850,
+ 891,
+ 927,
+ 976,
+ 1038,
+ 1095,
+ 1147,
+ 1194,
+ 1244,
+ 1297,
+ 1346,
+ 1391,
+ 1655
+ ],
+ [
+ 436,
+ 598,
+ 677,
+ 743,
+ 800,
+ 851,
+ 897,
+ 939,
+ 978,
+ 1030,
+ 1095,
+ 1155,
+ 1210,
+ 1260,
+ 1312,
+ 1367,
+ 1419,
+ 1467,
+ 1745
+ ],
+ [
+ 469,
+ 643,
+ 727,
+ 798,
+ 860,
+ 914,
+ 964,
+ 1008,
+ 1051,
+ 1106,
+ 1176,
+ 1240,
+ 1298,
+ 1353,
+ 1409,
+ 1467,
+ 1522,
+ 1575,
+ 1873
+ ]
+ ],
+ "ME": [
+ [
+ 305,
+ 413,
+ 465,
+ 509,
+ 546,
+ 580,
+ 609,
+ 637,
+ 663,
+ 696,
+ 738,
+ 777,
+ 812,
+ 845,
+ 878,
+ 914,
+ 947,
+ 978,
+ 1155
+ ],
+ [
+ 346,
+ 468,
+ 528,
+ 577,
+ 619,
+ 658,
+ 692,
+ 723,
+ 752,
+ 789,
+ 837,
+ 882,
+ 921,
+ 958,
+ 995,
+ 1036,
+ 1073,
+ 1108,
+ 1308
+ ],
+ [
+ 374,
+ 506,
+ 570,
+ 623,
+ 669,
+ 709,
+ 746,
+ 780,
+ 810,
+ 851,
+ 903,
+ 950,
+ 993,
+ 1033,
+ 1073,
+ 1117,
+ 1157,
+ 1194,
+ 1411
+ ],
+ [
+ 395,
+ 534,
+ 601,
+ 658,
+ 706,
+ 749,
+ 788,
+ 824,
+ 856,
+ 899,
+ 954,
+ 1003,
+ 1048,
+ 1091,
+ 1133,
+ 1179,
+ 1222,
+ 1261,
+ 1489
+ ],
+ [
+ 412,
+ 558,
+ 628,
+ 687,
+ 737,
+ 782,
+ 823,
+ 859,
+ 893,
+ 937,
+ 995,
+ 1047,
+ 1094,
+ 1137,
+ 1183,
+ 1230,
+ 1275,
+ 1316,
+ 1554
+ ],
+ [
+ 437,
+ 590,
+ 665,
+ 727,
+ 780,
+ 828,
+ 870,
+ 909,
+ 945,
+ 992,
+ 1053,
+ 1108,
+ 1158,
+ 1203,
+ 1251,
+ 1302,
+ 1349,
+ 1392,
+ 1643
+ ]
+ ],
+ "MI": [
+ [
+ 342,
+ 470,
+ 533,
+ 586,
+ 632,
+ 672,
+ 709,
+ 742,
+ 774,
+ 816,
+ 867,
+ 915,
+ 959,
+ 999,
+ 1042,
+ 1086,
+ 1127,
+ 1166,
+ 1389
+ ],
+ [
+ 392,
+ 540,
+ 612,
+ 673,
+ 726,
+ 773,
+ 814,
+ 853,
+ 890,
+ 936,
+ 996,
+ 1052,
+ 1102,
+ 1149,
+ 1196,
+ 1247,
+ 1295,
+ 1340,
+ 1596
+ ],
+ [
+ 427,
+ 587,
+ 666,
+ 731,
+ 789,
+ 840,
+ 886,
+ 927,
+ 967,
+ 1018,
+ 1084,
+ 1144,
+ 1197,
+ 1248,
+ 1301,
+ 1356,
+ 1408,
+ 1456,
+ 1736
+ ],
+ [
+ 453,
+ 624,
+ 707,
+ 777,
+ 838,
+ 892,
+ 940,
+ 985,
+ 1027,
+ 1081,
+ 1151,
+ 1214,
+ 1271,
+ 1326,
+ 1381,
+ 1440,
+ 1495,
+ 1547,
+ 1843
+ ],
+ [
+ 474,
+ 653,
+ 741,
+ 814,
+ 878,
+ 934,
+ 986,
+ 1033,
+ 1075,
+ 1133,
+ 1206,
+ 1273,
+ 1333,
+ 1390,
+ 1448,
+ 1510,
+ 1567,
+ 1622,
+ 1933
+ ],
+ [
+ 505,
+ 696,
+ 789,
+ 867,
+ 935,
+ 995,
+ 1049,
+ 1099,
+ 1146,
+ 1206,
+ 1284,
+ 1355,
+ 1420,
+ 1480,
+ 1542,
+ 1608,
+ 1669,
+ 1726,
+ 2058
+ ]
+ ],
+ "MN": [
+ [
+ 366,
+ 513,
+ 586,
+ 648,
+ 702,
+ 749,
+ 793,
+ 833,
+ 870,
+ 919,
+ 982,
+ 1040,
+ 1093,
+ 1142,
+ 1192,
+ 1246,
+ 1297,
+ 1345,
+ 1619
+ ],
+ [
+ 414,
+ 582,
+ 665,
+ 735,
+ 796,
+ 851,
+ 900,
+ 945,
+ 988,
+ 1044,
+ 1115,
+ 1181,
+ 1241,
+ 1297,
+ 1355,
+ 1416,
+ 1473,
+ 1527,
+ 1840
+ ],
+ [
+ 447,
+ 628,
+ 717,
+ 792,
+ 858,
+ 917,
+ 971,
+ 1020,
+ 1065,
+ 1125,
+ 1203,
+ 1274,
+ 1339,
+ 1399,
+ 1461,
+ 1527,
+ 1589,
+ 1648,
+ 1986
+ ],
+ [
+ 471,
+ 662,
+ 757,
+ 837,
+ 907,
+ 969,
+ 1025,
+ 1077,
+ 1125,
+ 1189,
+ 1270,
+ 1346,
+ 1414,
+ 1478,
+ 1544,
+ 1614,
+ 1679,
+ 1741,
+ 2098
+ ],
+ [
+ 492,
+ 691,
+ 790,
+ 873,
+ 945,
+ 1010,
+ 1069,
+ 1124,
+ 1174,
+ 1241,
+ 1325,
+ 1405,
+ 1476,
+ 1543,
+ 1611,
+ 1684,
+ 1752,
+ 1817,
+ 2191
+ ],
+ [
+ 520,
+ 730,
+ 836,
+ 923,
+ 1000,
+ 1069,
+ 1131,
+ 1189,
+ 1242,
+ 1313,
+ 1403,
+ 1486,
+ 1561,
+ 1632,
+ 1705,
+ 1782,
+ 1854,
+ 1923,
+ 2319
+ ]
+ ],
+ "MO": [
+ [
+ 263,
+ 363,
+ 411,
+ 452,
+ 487,
+ 519,
+ 547,
+ 574,
+ 598,
+ 630,
+ 670,
+ 708,
+ 741,
+ 773,
+ 805,
+ 840,
+ 872,
+ 903,
+ 1077
+ ],
+ [
+ 307,
+ 423,
+ 480,
+ 528,
+ 570,
+ 606,
+ 640,
+ 670,
+ 699,
+ 736,
+ 784,
+ 828,
+ 867,
+ 904,
+ 942,
+ 982,
+ 1020,
+ 1055,
+ 1259
+ ],
+ [
+ 337,
+ 465,
+ 528,
+ 580,
+ 626,
+ 666,
+ 703,
+ 736,
+ 768,
+ 808,
+ 861,
+ 909,
+ 953,
+ 993,
+ 1035,
+ 1080,
+ 1121,
+ 1160,
+ 1384
+ ],
+ [
+ 361,
+ 498,
+ 565,
+ 621,
+ 670,
+ 713,
+ 753,
+ 788,
+ 822,
+ 865,
+ 922,
+ 973,
+ 1020,
+ 1063,
+ 1108,
+ 1156,
+ 1199,
+ 1242,
+ 1482
+ ],
+ [
+ 381,
+ 525,
+ 596,
+ 655,
+ 707,
+ 753,
+ 793,
+ 832,
+ 866,
+ 913,
+ 972,
+ 1027,
+ 1075,
+ 1121,
+ 1169,
+ 1219,
+ 1265,
+ 1310,
+ 1563
+ ],
+ [
+ 408,
+ 563,
+ 639,
+ 703,
+ 759,
+ 807,
+ 852,
+ 893,
+ 930,
+ 980,
+ 1044,
+ 1102,
+ 1155,
+ 1203,
+ 1254,
+ 1308,
+ 1359,
+ 1406,
+ 1678
+ ]
+ ],
+ "MS": [
+ [
+ 611,
+ 819,
+ 918,
+ 1000,
+ 1071,
+ 1134,
+ 1191,
+ 1243,
+ 1290,
+ 1353,
+ 1432,
+ 1504,
+ 1570,
+ 1630,
+ 1693,
+ 1759,
+ 1820,
+ 1877,
+ 2205
+ ],
+ [
+ 741,
+ 991,
+ 1111,
+ 1210,
+ 1295,
+ 1371,
+ 1438,
+ 1500,
+ 1558,
+ 1632,
+ 1727,
+ 1815,
+ 1893,
+ 1966,
+ 2040,
+ 2119,
+ 2193,
+ 2262,
+ 2653
+ ],
+ [
+ 833,
+ 1111,
+ 1245,
+ 1356,
+ 1451,
+ 1536,
+ 1611,
+ 1680,
+ 1744,
+ 1827,
+ 1934,
+ 2030,
+ 2117,
+ 2199,
+ 2282,
+ 2370,
+ 2452,
+ 2529,
+ 2964
+ ],
+ [
+ 904,
+ 1206,
+ 1351,
+ 1471,
+ 1574,
+ 1666,
+ 1748,
+ 1822,
+ 1892,
+ 1982,
+ 2097,
+ 2202,
+ 2296,
+ 2385,
+ 2474,
+ 2569,
+ 2658,
+ 2741,
+ 3212
+ ],
+ [
+ 965,
+ 1286,
+ 1441,
+ 1568,
+ 1679,
+ 1775,
+ 1863,
+ 1943,
+ 2016,
+ 2112,
+ 2234,
+ 2346,
+ 2447,
+ 2540,
+ 2635,
+ 2737,
+ 2830,
+ 2919,
+ 3420
+ ],
+ [
+ 1051,
+ 1400,
+ 1568,
+ 1707,
+ 1827,
+ 1932,
+ 2027,
+ 2113,
+ 2194,
+ 2297,
+ 2430,
+ 2551,
+ 2661,
+ 2762,
+ 2866,
+ 2976,
+ 3078,
+ 3174,
+ 3718
+ ]
+ ],
+ "MT": [
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ ],
+ "NC": [
+ [
+ 345,
+ 465,
+ 522,
+ 570,
+ 611,
+ 648,
+ 680,
+ 711,
+ 738,
+ 774,
+ 821,
+ 862,
+ 901,
+ 936,
+ 973,
+ 1010,
+ 1046,
+ 1080,
+ 1271
+ ],
+ [
+ 413,
+ 555,
+ 624,
+ 680,
+ 729,
+ 772,
+ 811,
+ 847,
+ 879,
+ 923,
+ 977,
+ 1028,
+ 1072,
+ 1115,
+ 1158,
+ 1203,
+ 1245,
+ 1286,
+ 1512
+ ],
+ [
+ 460,
+ 617,
+ 694,
+ 757,
+ 810,
+ 859,
+ 902,
+ 941,
+ 978,
+ 1026,
+ 1087,
+ 1142,
+ 1192,
+ 1238,
+ 1287,
+ 1336,
+ 1383,
+ 1428,
+ 1679
+ ],
+ [
+ 497,
+ 667,
+ 748,
+ 817,
+ 875,
+ 927,
+ 974,
+ 1017,
+ 1055,
+ 1107,
+ 1172,
+ 1232,
+ 1286,
+ 1336,
+ 1388,
+ 1442,
+ 1493,
+ 1541,
+ 1811
+ ],
+ [
+ 528,
+ 709,
+ 795,
+ 867,
+ 929,
+ 984,
+ 1034,
+ 1080,
+ 1121,
+ 1175,
+ 1245,
+ 1308,
+ 1365,
+ 1419,
+ 1474,
+ 1530,
+ 1584,
+ 1635,
+ 1921
+ ],
+ [
+ 573,
+ 768,
+ 862,
+ 939,
+ 1007,
+ 1066,
+ 1120,
+ 1169,
+ 1214,
+ 1273,
+ 1348,
+ 1417,
+ 1479,
+ 1536,
+ 1594,
+ 1657,
+ 1715,
+ 1769,
+ 2079
+ ]
+ ],
+ "ND": [
+ [
+ 284,
+ 391,
+ 444,
+ 488,
+ 526,
+ 561,
+ 591,
+ 619,
+ 646,
+ 679,
+ 724,
+ 764,
+ 800,
+ 835,
+ 869,
+ 907,
+ 941,
+ 974,
+ 1162
+ ],
+ [
+ 330,
+ 455,
+ 516,
+ 567,
+ 611,
+ 650,
+ 686,
+ 718,
+ 748,
+ 788,
+ 839,
+ 886,
+ 928,
+ 968,
+ 1008,
+ 1051,
+ 1091,
+ 1129,
+ 1346
+ ],
+ [
+ 362,
+ 498,
+ 565,
+ 619,
+ 668,
+ 712,
+ 751,
+ 786,
+ 820,
+ 862,
+ 918,
+ 969,
+ 1016,
+ 1058,
+ 1103,
+ 1150,
+ 1193,
+ 1235,
+ 1472
+ ],
+ [
+ 386,
+ 531,
+ 602,
+ 662,
+ 714,
+ 760,
+ 801,
+ 839,
+ 874,
+ 920,
+ 980,
+ 1034,
+ 1084,
+ 1129,
+ 1176,
+ 1227,
+ 1274,
+ 1317,
+ 1570
+ ],
+ [
+ 406,
+ 560,
+ 634,
+ 697,
+ 752,
+ 799,
+ 843,
+ 884,
+ 920,
+ 969,
+ 1031,
+ 1089,
+ 1140,
+ 1188,
+ 1238,
+ 1291,
+ 1340,
+ 1386,
+ 1651
+ ],
+ [
+ 435,
+ 599,
+ 679,
+ 745,
+ 804,
+ 856,
+ 903,
+ 945,
+ 985,
+ 1037,
+ 1104,
+ 1165,
+ 1220,
+ 1271,
+ 1325,
+ 1381,
+ 1434,
+ 1484,
+ 1767
+ ]
+ ],
+ "NE": [
+ [
+ 339,
+ 474,
+ 541,
+ 598,
+ 647,
+ 691,
+ 730,
+ 767,
+ 801,
+ 846,
+ 903,
+ 956,
+ 1003,
+ 1048,
+ 1095,
+ 1144,
+ 1189,
+ 1233,
+ 1483
+ ],
+ [
+ 389,
+ 544,
+ 622,
+ 687,
+ 742,
+ 793,
+ 839,
+ 880,
+ 920,
+ 971,
+ 1037,
+ 1098,
+ 1153,
+ 1204,
+ 1257,
+ 1313,
+ 1366,
+ 1416,
+ 1703
+ ],
+ [
+ 422,
+ 591,
+ 675,
+ 745,
+ 806,
+ 861,
+ 911,
+ 957,
+ 999,
+ 1055,
+ 1126,
+ 1192,
+ 1252,
+ 1308,
+ 1366,
+ 1427,
+ 1484,
+ 1539,
+ 1850
+ ],
+ [
+ 448,
+ 628,
+ 716,
+ 791,
+ 856,
+ 914,
+ 967,
+ 1016,
+ 1060,
+ 1119,
+ 1195,
+ 1265,
+ 1328,
+ 1388,
+ 1449,
+ 1514,
+ 1575,
+ 1633,
+ 1963
+ ],
+ [
+ 469,
+ 657,
+ 751,
+ 829,
+ 897,
+ 958,
+ 1013,
+ 1063,
+ 1111,
+ 1173,
+ 1252,
+ 1325,
+ 1392,
+ 1454,
+ 1518,
+ 1586,
+ 1650,
+ 1710,
+ 2058
+ ],
+ [
+ 499,
+ 699,
+ 798,
+ 880,
+ 954,
+ 1019,
+ 1078,
+ 1131,
+ 1181,
+ 1247,
+ 1332,
+ 1410,
+ 1481,
+ 1547,
+ 1615,
+ 1688,
+ 1755,
+ 1819,
+ 2189
+ ]
+ ],
+ "NH": [
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ ],
+ "NJ": [
+ [
+ 402,
+ 556,
+ 631,
+ 694,
+ 748,
+ 797,
+ 842,
+ 883,
+ 920,
+ 969,
+ 1032,
+ 1090,
+ 1143,
+ 1191,
+ 1242,
+ 1296,
+ 1346,
+ 1392,
+ 1664
+ ],
+ [
+ 451,
+ 624,
+ 708,
+ 779,
+ 841,
+ 896,
+ 944,
+ 990,
+ 1033,
+ 1088,
+ 1159,
+ 1224,
+ 1283,
+ 1339,
+ 1394,
+ 1455,
+ 1511,
+ 1564,
+ 1868
+ ],
+ [
+ 483,
+ 667,
+ 759,
+ 835,
+ 901,
+ 960,
+ 1013,
+ 1061,
+ 1107,
+ 1166,
+ 1242,
+ 1312,
+ 1375,
+ 1434,
+ 1495,
+ 1559,
+ 1620,
+ 1677,
+ 2002
+ ],
+ [
+ 508,
+ 702,
+ 797,
+ 877,
+ 947,
+ 1008,
+ 1064,
+ 1115,
+ 1163,
+ 1226,
+ 1305,
+ 1378,
+ 1445,
+ 1507,
+ 1571,
+ 1639,
+ 1702,
+ 1762,
+ 2104
+ ],
+ [
+ 527,
+ 729,
+ 829,
+ 912,
+ 984,
+ 1048,
+ 1106,
+ 1160,
+ 1210,
+ 1274,
+ 1357,
+ 1433,
+ 1503,
+ 1567,
+ 1634,
+ 1704,
+ 1770,
+ 1832,
+ 2189
+ ],
+ [
+ 556,
+ 768,
+ 872,
+ 960,
+ 1036,
+ 1104,
+ 1165,
+ 1221,
+ 1273,
+ 1342,
+ 1429,
+ 1509,
+ 1582,
+ 1650,
+ 1720,
+ 1795,
+ 1864,
+ 1930,
+ 2304
+ ]
+ ],
+ "NM": [
+ [
+ 366,
+ 509,
+ 580,
+ 639,
+ 691,
+ 737,
+ 779,
+ 817,
+ 852,
+ 899,
+ 959,
+ 1015,
+ 1064,
+ 1111,
+ 1159,
+ 1210,
+ 1257,
+ 1302,
+ 1561
+ ],
+ [
+ 413,
+ 575,
+ 655,
+ 721,
+ 780,
+ 832,
+ 878,
+ 922,
+ 962,
+ 1015,
+ 1083,
+ 1145,
+ 1200,
+ 1253,
+ 1307,
+ 1365,
+ 1419,
+ 1470,
+ 1761
+ ],
+ [
+ 444,
+ 617,
+ 704,
+ 775,
+ 838,
+ 894,
+ 944,
+ 990,
+ 1034,
+ 1090,
+ 1163,
+ 1229,
+ 1290,
+ 1347,
+ 1405,
+ 1466,
+ 1524,
+ 1578,
+ 1892
+ ],
+ [
+ 467,
+ 650,
+ 740,
+ 817,
+ 883,
+ 940,
+ 994,
+ 1043,
+ 1088,
+ 1148,
+ 1224,
+ 1294,
+ 1358,
+ 1418,
+ 1479,
+ 1544,
+ 1605,
+ 1661,
+ 1992
+ ],
+ [
+ 486,
+ 676,
+ 771,
+ 849,
+ 918,
+ 979,
+ 1035,
+ 1086,
+ 1132,
+ 1194,
+ 1274,
+ 1347,
+ 1414,
+ 1476,
+ 1539,
+ 1607,
+ 1670,
+ 1730,
+ 2073
+ ],
+ [
+ 513,
+ 714,
+ 812,
+ 896,
+ 968,
+ 1033,
+ 1091,
+ 1145,
+ 1194,
+ 1259,
+ 1344,
+ 1420,
+ 1490,
+ 1555,
+ 1623,
+ 1694,
+ 1760,
+ 1823,
+ 2186
+ ]
+ ],
+ "NV": [
+ [
+ 392,
+ 536,
+ 607,
+ 665,
+ 716,
+ 762,
+ 802,
+ 839,
+ 873,
+ 919,
+ 977,
+ 1030,
+ 1078,
+ 1122,
+ 1169,
+ 1217,
+ 1262,
+ 1305,
+ 1550
+ ],
+ [
+ 453,
+ 619,
+ 701,
+ 768,
+ 827,
+ 878,
+ 925,
+ 968,
+ 1008,
+ 1060,
+ 1127,
+ 1188,
+ 1243,
+ 1295,
+ 1348,
+ 1404,
+ 1456,
+ 1505,
+ 1786
+ ],
+ [
+ 494,
+ 675,
+ 764,
+ 837,
+ 901,
+ 958,
+ 1008,
+ 1055,
+ 1099,
+ 1156,
+ 1229,
+ 1295,
+ 1355,
+ 1412,
+ 1469,
+ 1530,
+ 1586,
+ 1640,
+ 1947
+ ],
+ [
+ 525,
+ 718,
+ 812,
+ 891,
+ 959,
+ 1019,
+ 1073,
+ 1123,
+ 1169,
+ 1230,
+ 1307,
+ 1378,
+ 1442,
+ 1502,
+ 1563,
+ 1628,
+ 1689,
+ 1746,
+ 2072
+ ],
+ [
+ 551,
+ 755,
+ 853,
+ 935,
+ 1006,
+ 1069,
+ 1127,
+ 1179,
+ 1228,
+ 1291,
+ 1372,
+ 1446,
+ 1514,
+ 1577,
+ 1641,
+ 1709,
+ 1773,
+ 1833,
+ 2175
+ ],
+ [
+ 589,
+ 804,
+ 910,
+ 998,
+ 1074,
+ 1142,
+ 1202,
+ 1258,
+ 1310,
+ 1378,
+ 1464,
+ 1544,
+ 1615,
+ 1682,
+ 1751,
+ 1824,
+ 1891,
+ 1955,
+ 2321
+ ]
+ ],
+ "NY": [
+ [
+ 246,
+ 339,
+ 386,
+ 425,
+ 458,
+ 487,
+ 515,
+ 539,
+ 563,
+ 592,
+ 631,
+ 666,
+ 699,
+ 729,
+ 760,
+ 792,
+ 823,
+ 852,
+ 1018
+ ],
+ [
+ 274,
+ 379,
+ 431,
+ 473,
+ 511,
+ 544,
+ 575,
+ 602,
+ 628,
+ 661,
+ 705,
+ 744,
+ 780,
+ 813,
+ 848,
+ 885,
+ 919,
+ 952,
+ 1136
+ ],
+ [
+ 292,
+ 404,
+ 460,
+ 506,
+ 545,
+ 581,
+ 613,
+ 643,
+ 670,
+ 707,
+ 753,
+ 795,
+ 834,
+ 869,
+ 906,
+ 945,
+ 982,
+ 1016,
+ 1214
+ ],
+ [
+ 307,
+ 423,
+ 481,
+ 530,
+ 572,
+ 609,
+ 643,
+ 674,
+ 703,
+ 740,
+ 789,
+ 834,
+ 873,
+ 911,
+ 950,
+ 991,
+ 1029,
+ 1065,
+ 1273
+ ],
+ [
+ 318,
+ 440,
+ 500,
+ 549,
+ 593,
+ 633,
+ 667,
+ 700,
+ 729,
+ 769,
+ 819,
+ 865,
+ 907,
+ 945,
+ 986,
+ 1029,
+ 1068,
+ 1106,
+ 1321
+ ],
+ [
+ 334,
+ 462,
+ 525,
+ 577,
+ 624,
+ 664,
+ 701,
+ 734,
+ 766,
+ 807,
+ 860,
+ 908,
+ 952,
+ 993,
+ 1035,
+ 1080,
+ 1121,
+ 1161,
+ 1387
+ ]
+ ],
+ "OH": [
+ [
+ 372,
+ 513,
+ 583,
+ 641,
+ 692,
+ 736,
+ 777,
+ 814,
+ 849,
+ 894,
+ 953,
+ 1005,
+ 1054,
+ 1099,
+ 1146,
+ 1194,
+ 1240,
+ 1284,
+ 1532
+ ],
+ [
+ 420,
+ 580,
+ 659,
+ 724,
+ 782,
+ 832,
+ 878,
+ 920,
+ 960,
+ 1010,
+ 1077,
+ 1136,
+ 1191,
+ 1242,
+ 1295,
+ 1351,
+ 1403,
+ 1451,
+ 1733
+ ],
+ [
+ 452,
+ 625,
+ 709,
+ 780,
+ 841,
+ 896,
+ 945,
+ 990,
+ 1033,
+ 1088,
+ 1159,
+ 1223,
+ 1282,
+ 1336,
+ 1393,
+ 1453,
+ 1509,
+ 1562,
+ 1864
+ ],
+ [
+ 476,
+ 658,
+ 747,
+ 822,
+ 887,
+ 944,
+ 996,
+ 1044,
+ 1089,
+ 1147,
+ 1221,
+ 1290,
+ 1352,
+ 1410,
+ 1469,
+ 1531,
+ 1590,
+ 1646,
+ 1965
+ ],
+ [
+ 497,
+ 686,
+ 779,
+ 856,
+ 924,
+ 984,
+ 1038,
+ 1088,
+ 1134,
+ 1195,
+ 1273,
+ 1344,
+ 1409,
+ 1469,
+ 1530,
+ 1596,
+ 1657,
+ 1715,
+ 2047
+ ],
+ [
+ 525,
+ 724,
+ 823,
+ 905,
+ 976,
+ 1039,
+ 1097,
+ 1150,
+ 1198,
+ 1262,
+ 1345,
+ 1420,
+ 1488,
+ 1552,
+ 1617,
+ 1686,
+ 1751,
+ 1812,
+ 2163
+ ]
+ ],
+ "OK": [
+ [
+ 318,
+ 432,
+ 487,
+ 534,
+ 574,
+ 609,
+ 642,
+ 671,
+ 699,
+ 734,
+ 780,
+ 822,
+ 859,
+ 894,
+ 930,
+ 968,
+ 1003,
+ 1037,
+ 1229
+ ],
+ [
+ 381,
+ 517,
+ 584,
+ 639,
+ 687,
+ 729,
+ 768,
+ 802,
+ 835,
+ 877,
+ 931,
+ 981,
+ 1026,
+ 1067,
+ 1110,
+ 1156,
+ 1198,
+ 1238,
+ 1465
+ ],
+ [
+ 425,
+ 577,
+ 651,
+ 712,
+ 766,
+ 812,
+ 855,
+ 894,
+ 929,
+ 977,
+ 1037,
+ 1092,
+ 1142,
+ 1188,
+ 1236,
+ 1287,
+ 1333,
+ 1377,
+ 1630
+ ],
+ [
+ 459,
+ 624,
+ 704,
+ 770,
+ 828,
+ 878,
+ 924,
+ 966,
+ 1005,
+ 1056,
+ 1121,
+ 1180,
+ 1234,
+ 1285,
+ 1335,
+ 1390,
+ 1441,
+ 1489,
+ 1761
+ ],
+ [
+ 488,
+ 663,
+ 748,
+ 819,
+ 880,
+ 934,
+ 983,
+ 1028,
+ 1068,
+ 1123,
+ 1192,
+ 1255,
+ 1312,
+ 1365,
+ 1420,
+ 1478,
+ 1531,
+ 1582,
+ 1871
+ ],
+ [
+ 531,
+ 720,
+ 812,
+ 889,
+ 955,
+ 1014,
+ 1066,
+ 1114,
+ 1159,
+ 1218,
+ 1293,
+ 1361,
+ 1423,
+ 1481,
+ 1540,
+ 1602,
+ 1660,
+ 1715,
+ 2028
+ ]
+ ],
+ "OR": [
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ ],
+ "PA": [
+ [
+ 351,
+ 474,
+ 534,
+ 583,
+ 626,
+ 664,
+ 698,
+ 729,
+ 758,
+ 795,
+ 843,
+ 887,
+ 926,
+ 963,
+ 1000,
+ 1041,
+ 1078,
+ 1112,
+ 1311
+ ],
+ [
+ 397,
+ 535,
+ 602,
+ 658,
+ 706,
+ 747,
+ 786,
+ 821,
+ 853,
+ 896,
+ 950,
+ 998,
+ 1043,
+ 1085,
+ 1127,
+ 1172,
+ 1214,
+ 1252,
+ 1477
+ ],
+ [
+ 427,
+ 575,
+ 647,
+ 707,
+ 758,
+ 803,
+ 844,
+ 882,
+ 916,
+ 962,
+ 1020,
+ 1072,
+ 1120,
+ 1165,
+ 1210,
+ 1258,
+ 1303,
+ 1345,
+ 1584
+ ],
+ [
+ 449,
+ 605,
+ 681,
+ 743,
+ 798,
+ 846,
+ 889,
+ 928,
+ 965,
+ 1013,
+ 1073,
+ 1128,
+ 1179,
+ 1226,
+ 1274,
+ 1324,
+ 1371,
+ 1416,
+ 1668
+ ],
+ [
+ 468,
+ 631,
+ 709,
+ 775,
+ 831,
+ 880,
+ 925,
+ 967,
+ 1004,
+ 1054,
+ 1117,
+ 1175,
+ 1227,
+ 1276,
+ 1325,
+ 1378,
+ 1427,
+ 1474,
+ 1736
+ ],
+ [
+ 494,
+ 665,
+ 748,
+ 818,
+ 876,
+ 929,
+ 976,
+ 1020,
+ 1059,
+ 1112,
+ 1178,
+ 1239,
+ 1294,
+ 1346,
+ 1397,
+ 1453,
+ 1505,
+ 1554,
+ 1830
+ ]
+ ],
+ "RI": [
+ [
+ 370,
+ 507,
+ 575,
+ 631,
+ 678,
+ 722,
+ 761,
+ 796,
+ 830,
+ 872,
+ 928,
+ 979,
+ 1025,
+ 1067,
+ 1112,
+ 1159,
+ 1201,
+ 1243,
+ 1478
+ ],
+ [
+ 423,
+ 581,
+ 657,
+ 721,
+ 777,
+ 826,
+ 870,
+ 911,
+ 949,
+ 998,
+ 1061,
+ 1119,
+ 1172,
+ 1221,
+ 1271,
+ 1324,
+ 1374,
+ 1421,
+ 1689
+ ],
+ [
+ 460,
+ 630,
+ 713,
+ 782,
+ 842,
+ 896,
+ 943,
+ 988,
+ 1029,
+ 1083,
+ 1151,
+ 1214,
+ 1270,
+ 1323,
+ 1378,
+ 1436,
+ 1489,
+ 1540,
+ 1830
+ ],
+ [
+ 487,
+ 668,
+ 756,
+ 830,
+ 893,
+ 950,
+ 1000,
+ 1047,
+ 1091,
+ 1148,
+ 1220,
+ 1286,
+ 1347,
+ 1403,
+ 1460,
+ 1521,
+ 1578,
+ 1632,
+ 1939
+ ],
+ [
+ 511,
+ 699,
+ 791,
+ 868,
+ 935,
+ 994,
+ 1047,
+ 1096,
+ 1142,
+ 1200,
+ 1277,
+ 1347,
+ 1410,
+ 1469,
+ 1528,
+ 1592,
+ 1651,
+ 1708,
+ 2029
+ ],
+ [
+ 543,
+ 743,
+ 841,
+ 922,
+ 993,
+ 1056,
+ 1113,
+ 1165,
+ 1213,
+ 1276,
+ 1357,
+ 1430,
+ 1497,
+ 1560,
+ 1624,
+ 1692,
+ 1755,
+ 1814,
+ 2155
+ ]
+ ],
+ "SC": [
+ [
+ 351,
+ 484,
+ 549,
+ 604,
+ 652,
+ 694,
+ 732,
+ 767,
+ 799,
+ 842,
+ 897,
+ 947,
+ 991,
+ 1034,
+ 1078,
+ 1123,
+ 1166,
+ 1206,
+ 1439
+ ],
+ [
+ 402,
+ 555,
+ 629,
+ 692,
+ 745,
+ 794,
+ 838,
+ 877,
+ 914,
+ 963,
+ 1026,
+ 1083,
+ 1134,
+ 1182,
+ 1232,
+ 1285,
+ 1333,
+ 1380,
+ 1645
+ ],
+ [
+ 437,
+ 601,
+ 682,
+ 749,
+ 808,
+ 860,
+ 908,
+ 951,
+ 991,
+ 1044,
+ 1111,
+ 1173,
+ 1229,
+ 1281,
+ 1334,
+ 1392,
+ 1445,
+ 1495,
+ 1782
+ ],
+ [
+ 462,
+ 637,
+ 723,
+ 794,
+ 857,
+ 912,
+ 962,
+ 1007,
+ 1050,
+ 1106,
+ 1177,
+ 1242,
+ 1302,
+ 1357,
+ 1415,
+ 1475,
+ 1530,
+ 1584,
+ 1888
+ ],
+ [
+ 484,
+ 667,
+ 757,
+ 832,
+ 897,
+ 955,
+ 1006,
+ 1054,
+ 1099,
+ 1157,
+ 1232,
+ 1300,
+ 1362,
+ 1420,
+ 1480,
+ 1543,
+ 1602,
+ 1657,
+ 1975
+ ],
+ [
+ 514,
+ 708,
+ 803,
+ 883,
+ 952,
+ 1014,
+ 1068,
+ 1119,
+ 1167,
+ 1229,
+ 1308,
+ 1380,
+ 1446,
+ 1508,
+ 1571,
+ 1638,
+ 1700,
+ 1759,
+ 2097
+ ]
+ ],
+ "SD": [
+ [
+ 381,
+ 519,
+ 586,
+ 642,
+ 691,
+ 734,
+ 773,
+ 808,
+ 841,
+ 885,
+ 939,
+ 990,
+ 1036,
+ 1079,
+ 1122,
+ 1169,
+ 1212,
+ 1252,
+ 1485
+ ],
+ [
+ 460,
+ 626,
+ 707,
+ 775,
+ 833,
+ 885,
+ 931,
+ 974,
+ 1014,
+ 1065,
+ 1132,
+ 1192,
+ 1247,
+ 1299,
+ 1352,
+ 1407,
+ 1458,
+ 1507,
+ 1786
+ ],
+ [
+ 515,
+ 701,
+ 791,
+ 866,
+ 931,
+ 989,
+ 1041,
+ 1090,
+ 1133,
+ 1191,
+ 1265,
+ 1333,
+ 1394,
+ 1451,
+ 1510,
+ 1572,
+ 1630,
+ 1684,
+ 1996
+ ],
+ [
+ 558,
+ 760,
+ 857,
+ 939,
+ 1009,
+ 1072,
+ 1128,
+ 1180,
+ 1228,
+ 1291,
+ 1371,
+ 1444,
+ 1510,
+ 1572,
+ 1636,
+ 1703,
+ 1765,
+ 1824,
+ 2160
+ ],
+ [
+ 595,
+ 809,
+ 913,
+ 1000,
+ 1075,
+ 1142,
+ 1201,
+ 1257,
+ 1308,
+ 1375,
+ 1459,
+ 1538,
+ 1609,
+ 1674,
+ 1742,
+ 1813,
+ 1879,
+ 1942,
+ 2299
+ ],
+ [
+ 647,
+ 879,
+ 993,
+ 1087,
+ 1169,
+ 1241,
+ 1306,
+ 1366,
+ 1421,
+ 1494,
+ 1586,
+ 1671,
+ 1747,
+ 1819,
+ 1892,
+ 1969,
+ 2041,
+ 2109,
+ 2497
+ ]
+ ],
+ "TN": [
+ [
+ 537,
+ 728,
+ 822,
+ 898,
+ 965,
+ 1024,
+ 1077,
+ 1125,
+ 1171,
+ 1230,
+ 1305,
+ 1373,
+ 1436,
+ 1494,
+ 1553,
+ 1616,
+ 1675,
+ 1730,
+ 2044
+ ],
+ [
+ 640,
+ 866,
+ 976,
+ 1067,
+ 1147,
+ 1217,
+ 1280,
+ 1338,
+ 1391,
+ 1460,
+ 1550,
+ 1631,
+ 1705,
+ 1774,
+ 1844,
+ 1918,
+ 1988,
+ 2053,
+ 2425
+ ],
+ [
+ 710,
+ 962,
+ 1084,
+ 1184,
+ 1273,
+ 1350,
+ 1420,
+ 1484,
+ 1543,
+ 1620,
+ 1719,
+ 1809,
+ 1891,
+ 1967,
+ 2045,
+ 2128,
+ 2204,
+ 2277,
+ 2689
+ ],
+ [
+ 766,
+ 1036,
+ 1168,
+ 1277,
+ 1371,
+ 1454,
+ 1529,
+ 1599,
+ 1662,
+ 1746,
+ 1852,
+ 1949,
+ 2037,
+ 2119,
+ 2203,
+ 2292,
+ 2374,
+ 2453,
+ 2896
+ ],
+ [
+ 811,
+ 1099,
+ 1238,
+ 1354,
+ 1454,
+ 1543,
+ 1622,
+ 1695,
+ 1763,
+ 1851,
+ 1964,
+ 2067,
+ 2160,
+ 2246,
+ 2336,
+ 2429,
+ 2518,
+ 2600,
+ 3070
+ ],
+ [
+ 877,
+ 1188,
+ 1339,
+ 1463,
+ 1572,
+ 1667,
+ 1753,
+ 1832,
+ 1905,
+ 2001,
+ 2123,
+ 2233,
+ 2334,
+ 2427,
+ 2524,
+ 2625,
+ 2720,
+ 2809,
+ 3316
+ ]
+ ],
+ "TX": [
+ [
+ 388,
+ 535,
+ 607,
+ 668,
+ 721,
+ 768,
+ 809,
+ 849,
+ 885,
+ 931,
+ 992,
+ 1047,
+ 1098,
+ 1145,
+ 1193,
+ 1244,
+ 1292,
+ 1336,
+ 1595
+ ],
+ [
+ 452,
+ 624,
+ 708,
+ 778,
+ 839,
+ 893,
+ 942,
+ 987,
+ 1029,
+ 1084,
+ 1155,
+ 1219,
+ 1277,
+ 1331,
+ 1387,
+ 1447,
+ 1502,
+ 1555,
+ 1854
+ ],
+ [
+ 496,
+ 682,
+ 775,
+ 852,
+ 919,
+ 978,
+ 1032,
+ 1082,
+ 1127,
+ 1187,
+ 1264,
+ 1334,
+ 1398,
+ 1458,
+ 1519,
+ 1584,
+ 1645,
+ 1702,
+ 2030
+ ],
+ [
+ 529,
+ 729,
+ 828,
+ 910,
+ 981,
+ 1045,
+ 1102,
+ 1155,
+ 1203,
+ 1267,
+ 1350,
+ 1425,
+ 1493,
+ 1557,
+ 1622,
+ 1691,
+ 1756,
+ 1817,
+ 2167
+ ],
+ [
+ 557,
+ 768,
+ 871,
+ 958,
+ 1033,
+ 1100,
+ 1160,
+ 1216,
+ 1266,
+ 1334,
+ 1421,
+ 1500,
+ 1571,
+ 1638,
+ 1707,
+ 1780,
+ 1848,
+ 1912,
+ 2281
+ ],
+ [
+ 596,
+ 823,
+ 933,
+ 1026,
+ 1106,
+ 1177,
+ 1242,
+ 1301,
+ 1356,
+ 1428,
+ 1520,
+ 1605,
+ 1682,
+ 1754,
+ 1827,
+ 1905,
+ 1978,
+ 2046,
+ 2440
+ ]
+ ],
+ "UT": [
+ [
+ 375,
+ 508,
+ 573,
+ 627,
+ 673,
+ 715,
+ 753,
+ 786,
+ 819,
+ 859,
+ 912,
+ 961,
+ 1004,
+ 1045,
+ 1087,
+ 1130,
+ 1172,
+ 1211,
+ 1432
+ ],
+ [
+ 445,
+ 603,
+ 680,
+ 744,
+ 799,
+ 848,
+ 893,
+ 933,
+ 970,
+ 1020,
+ 1082,
+ 1138,
+ 1190,
+ 1239,
+ 1288,
+ 1341,
+ 1388,
+ 1435,
+ 1696
+ ],
+ [
+ 493,
+ 668,
+ 754,
+ 825,
+ 886,
+ 939,
+ 989,
+ 1034,
+ 1074,
+ 1129,
+ 1198,
+ 1261,
+ 1318,
+ 1372,
+ 1427,
+ 1485,
+ 1538,
+ 1588,
+ 1878
+ ],
+ [
+ 531,
+ 720,
+ 811,
+ 888,
+ 954,
+ 1012,
+ 1064,
+ 1113,
+ 1157,
+ 1216,
+ 1290,
+ 1358,
+ 1420,
+ 1477,
+ 1536,
+ 1597,
+ 1655,
+ 1710,
+ 2021
+ ],
+ [
+ 563,
+ 763,
+ 860,
+ 940,
+ 1010,
+ 1072,
+ 1128,
+ 1179,
+ 1226,
+ 1288,
+ 1367,
+ 1438,
+ 1504,
+ 1564,
+ 1627,
+ 1692,
+ 1754,
+ 1812,
+ 2141
+ ],
+ [
+ 608,
+ 824,
+ 929,
+ 1016,
+ 1091,
+ 1158,
+ 1218,
+ 1273,
+ 1323,
+ 1390,
+ 1476,
+ 1553,
+ 1623,
+ 1689,
+ 1756,
+ 1827,
+ 1893,
+ 1955,
+ 2310
+ ]
+ ],
+ "VA": [
+ [
+ 241,
+ 332,
+ 377,
+ 414,
+ 447,
+ 475,
+ 502,
+ 525,
+ 547,
+ 577,
+ 613,
+ 648,
+ 678,
+ 708,
+ 737,
+ 769,
+ 798,
+ 826,
+ 985
+ ],
+ [
+ 276,
+ 381,
+ 433,
+ 475,
+ 513,
+ 545,
+ 575,
+ 602,
+ 628,
+ 661,
+ 704,
+ 743,
+ 779,
+ 811,
+ 846,
+ 883,
+ 916,
+ 948,
+ 1129
+ ],
+ [
+ 301,
+ 414,
+ 470,
+ 516,
+ 557,
+ 592,
+ 625,
+ 655,
+ 682,
+ 719,
+ 765,
+ 807,
+ 846,
+ 883,
+ 919,
+ 959,
+ 995,
+ 1030,
+ 1227
+ ],
+ [
+ 319,
+ 440,
+ 499,
+ 548,
+ 591,
+ 630,
+ 663,
+ 696,
+ 724,
+ 763,
+ 812,
+ 857,
+ 899,
+ 936,
+ 976,
+ 1018,
+ 1056,
+ 1093,
+ 1303
+ ],
+ [
+ 334,
+ 461,
+ 523,
+ 575,
+ 619,
+ 659,
+ 696,
+ 729,
+ 760,
+ 800,
+ 851,
+ 899,
+ 941,
+ 982,
+ 1023,
+ 1066,
+ 1107,
+ 1146,
+ 1366
+ ],
+ [
+ 356,
+ 491,
+ 557,
+ 611,
+ 660,
+ 702,
+ 740,
+ 776,
+ 808,
+ 851,
+ 907,
+ 957,
+ 1002,
+ 1045,
+ 1089,
+ 1135,
+ 1179,
+ 1220,
+ 1453
+ ]
+ ],
+ "VT": [
+ [
+ 270,
+ 373,
+ 422,
+ 465,
+ 502,
+ 533,
+ 563,
+ 590,
+ 614,
+ 647,
+ 690,
+ 727,
+ 763,
+ 795,
+ 829,
+ 863,
+ 897,
+ 928,
+ 1106
+ ],
+ [
+ 292,
+ 403,
+ 457,
+ 503,
+ 542,
+ 577,
+ 608,
+ 638,
+ 664,
+ 700,
+ 745,
+ 786,
+ 825,
+ 859,
+ 896,
+ 933,
+ 969,
+ 1003,
+ 1196
+ ],
+ [
+ 306,
+ 421,
+ 478,
+ 526,
+ 568,
+ 604,
+ 638,
+ 667,
+ 696,
+ 733,
+ 780,
+ 824,
+ 863,
+ 900,
+ 937,
+ 978,
+ 1015,
+ 1050,
+ 1252
+ ],
+ [
+ 317,
+ 436,
+ 495,
+ 544,
+ 587,
+ 625,
+ 659,
+ 690,
+ 719,
+ 758,
+ 806,
+ 851,
+ 892,
+ 930,
+ 969,
+ 1010,
+ 1049,
+ 1086,
+ 1294
+ ],
+ [
+ 325,
+ 448,
+ 508,
+ 558,
+ 602,
+ 641,
+ 675,
+ 708,
+ 738,
+ 777,
+ 828,
+ 873,
+ 915,
+ 954,
+ 994,
+ 1037,
+ 1077,
+ 1113,
+ 1327
+ ],
+ [
+ 336,
+ 463,
+ 525,
+ 577,
+ 623,
+ 662,
+ 699,
+ 732,
+ 763,
+ 803,
+ 856,
+ 903,
+ 947,
+ 987,
+ 1028,
+ 1072,
+ 1113,
+ 1152,
+ 1373
+ ]
+ ],
+ "WA": [
+ [
+ 430,
+ 594,
+ 675,
+ 743,
+ 802,
+ 855,
+ 903,
+ 947,
+ 987,
+ 1040,
+ 1108,
+ 1171,
+ 1228,
+ 1281,
+ 1335,
+ 1393,
+ 1447,
+ 1498,
+ 1790
+ ],
+ [
+ 492,
+ 680,
+ 773,
+ 851,
+ 919,
+ 979,
+ 1033,
+ 1084,
+ 1129,
+ 1190,
+ 1268,
+ 1341,
+ 1406,
+ 1465,
+ 1528,
+ 1594,
+ 1656,
+ 1714,
+ 2049
+ ],
+ [
+ 532,
+ 737,
+ 838,
+ 922,
+ 996,
+ 1061,
+ 1120,
+ 1174,
+ 1225,
+ 1291,
+ 1375,
+ 1453,
+ 1523,
+ 1589,
+ 1657,
+ 1728,
+ 1796,
+ 1860,
+ 2222
+ ],
+ [
+ 565,
+ 782,
+ 889,
+ 978,
+ 1056,
+ 1125,
+ 1187,
+ 1245,
+ 1298,
+ 1368,
+ 1458,
+ 1541,
+ 1615,
+ 1685,
+ 1757,
+ 1833,
+ 1903,
+ 1970,
+ 2356
+ ],
+ [
+ 591,
+ 818,
+ 930,
+ 1024,
+ 1105,
+ 1177,
+ 1243,
+ 1303,
+ 1359,
+ 1432,
+ 1526,
+ 1612,
+ 1690,
+ 1763,
+ 1838,
+ 1918,
+ 1993,
+ 2063,
+ 2466
+ ],
+ [
+ 628,
+ 869,
+ 988,
+ 1088,
+ 1174,
+ 1251,
+ 1320,
+ 1384,
+ 1443,
+ 1521,
+ 1621,
+ 1712,
+ 1796,
+ 1874,
+ 1953,
+ 2038,
+ 2116,
+ 2192,
+ 2619
+ ]
+ ],
+ "WI": [
+ [
+ 328,
+ 454,
+ 516,
+ 568,
+ 613,
+ 653,
+ 690,
+ 723,
+ 755,
+ 795,
+ 847,
+ 895,
+ 938,
+ 979,
+ 1021,
+ 1064,
+ 1106,
+ 1145,
+ 1368
+ ],
+ [
+ 378,
+ 523,
+ 594,
+ 654,
+ 706,
+ 752,
+ 794,
+ 832,
+ 867,
+ 914,
+ 974,
+ 1029,
+ 1079,
+ 1125,
+ 1173,
+ 1224,
+ 1271,
+ 1316,
+ 1573
+ ],
+ [
+ 411,
+ 569,
+ 647,
+ 711,
+ 768,
+ 818,
+ 863,
+ 905,
+ 943,
+ 994,
+ 1059,
+ 1119,
+ 1173,
+ 1224,
+ 1276,
+ 1331,
+ 1382,
+ 1431,
+ 1710
+ ],
+ [
+ 437,
+ 604,
+ 687,
+ 756,
+ 816,
+ 869,
+ 917,
+ 962,
+ 1002,
+ 1056,
+ 1125,
+ 1189,
+ 1246,
+ 1300,
+ 1355,
+ 1414,
+ 1469,
+ 1520,
+ 1816
+ ],
+ [
+ 458,
+ 634,
+ 720,
+ 792,
+ 855,
+ 911,
+ 962,
+ 1008,
+ 1051,
+ 1108,
+ 1180,
+ 1246,
+ 1307,
+ 1363,
+ 1421,
+ 1482,
+ 1540,
+ 1593,
+ 1903
+ ],
+ [
+ 488,
+ 675,
+ 767,
+ 844,
+ 911,
+ 970,
+ 1024,
+ 1073,
+ 1119,
+ 1179,
+ 1256,
+ 1326,
+ 1391,
+ 1450,
+ 1512,
+ 1577,
+ 1638,
+ 1696,
+ 2026
+ ]
+ ],
+ "WV": [
+ [
+ 375,
+ 532,
+ 610,
+ 677,
+ 735,
+ 787,
+ 835,
+ 878,
+ 919,
+ 972,
+ 1041,
+ 1105,
+ 1163,
+ 1217,
+ 1273,
+ 1332,
+ 1388,
+ 1441,
+ 1747
+ ],
+ [
+ 435,
+ 617,
+ 709,
+ 786,
+ 854,
+ 915,
+ 970,
+ 1021,
+ 1068,
+ 1130,
+ 1211,
+ 1285,
+ 1352,
+ 1416,
+ 1481,
+ 1550,
+ 1615,
+ 1677,
+ 2033
+ ],
+ [
+ 475,
+ 675,
+ 776,
+ 860,
+ 935,
+ 1001,
+ 1061,
+ 1117,
+ 1169,
+ 1238,
+ 1325,
+ 1407,
+ 1481,
+ 1550,
+ 1621,
+ 1697,
+ 1768,
+ 1836,
+ 2227
+ ],
+ [
+ 507,
+ 720,
+ 828,
+ 918,
+ 997,
+ 1068,
+ 1132,
+ 1192,
+ 1248,
+ 1320,
+ 1415,
+ 1501,
+ 1580,
+ 1654,
+ 1731,
+ 1812,
+ 1888,
+ 1960,
+ 2377
+ ],
+ [
+ 533,
+ 758,
+ 870,
+ 966,
+ 1050,
+ 1124,
+ 1192,
+ 1254,
+ 1313,
+ 1389,
+ 1489,
+ 1580,
+ 1662,
+ 1741,
+ 1822,
+ 1907,
+ 1987,
+ 2063,
+ 2503
+ ],
+ [
+ 570,
+ 810,
+ 931,
+ 1033,
+ 1123,
+ 1202,
+ 1276,
+ 1343,
+ 1405,
+ 1487,
+ 1592,
+ 1690,
+ 1779,
+ 1863,
+ 1949,
+ 2040,
+ 2127,
+ 2208,
+ 2679
+ ]
+ ],
+ "WY": [
+ [
+ 247,
+ 343,
+ 391,
+ 431,
+ 465,
+ 496,
+ 524,
+ 549,
+ 573,
+ 604,
+ 645,
+ 681,
+ 715,
+ 745,
+ 778,
+ 812,
+ 844,
+ 874,
+ 1047
+ ],
+ [
+ 281,
+ 391,
+ 445,
+ 490,
+ 529,
+ 565,
+ 596,
+ 626,
+ 653,
+ 689,
+ 734,
+ 776,
+ 814,
+ 850,
+ 887,
+ 925,
+ 962,
+ 996,
+ 1193
+ ],
+ [
+ 304,
+ 421,
+ 480,
+ 529,
+ 572,
+ 610,
+ 645,
+ 676,
+ 706,
+ 744,
+ 793,
+ 839,
+ 880,
+ 918,
+ 959,
+ 1000,
+ 1040,
+ 1077,
+ 1290
+ ],
+ [
+ 321,
+ 446,
+ 508,
+ 560,
+ 605,
+ 645,
+ 681,
+ 715,
+ 746,
+ 787,
+ 839,
+ 887,
+ 930,
+ 972,
+ 1014,
+ 1058,
+ 1100,
+ 1138,
+ 1365
+ ],
+ [
+ 335,
+ 466,
+ 530,
+ 585,
+ 632,
+ 674,
+ 712,
+ 746,
+ 779,
+ 822,
+ 876,
+ 927,
+ 973,
+ 1015,
+ 1059,
+ 1106,
+ 1149,
+ 1190,
+ 1426
+ ],
+ [
+ 354,
+ 494,
+ 563,
+ 619,
+ 669,
+ 714,
+ 755,
+ 791,
+ 826,
+ 871,
+ 929,
+ 982,
+ 1031,
+ 1075,
+ 1122,
+ 1172,
+ 1218,
+ 1261,
+ 1511
+ ]
+ ]
+}
diff --git a/reference_audit/2026-09-28/verification/judge_verdicts.json b/reference_audit/2026-09-28/verification/judge_verdicts.json
index c56a1a29..37b2e650 100644
--- a/reference_audit/2026-09-28/verification/judge_verdicts.json
+++ b/reference_audit/2026-09-28/verification/judge_verdicts.json
@@ -20,10 +20,22 @@
"committed_on": "2026-09-30"
},
"2026-10-05": {
- "commit": null,
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
"release": "dashboard-data-20261006",
+ "pull_request": 202,
+ "committed_on": "2026-10-09"
+ },
+ "2026-10-06": {
+ "commit": null,
+ "release": "dashboard-data-20261010",
+ "pull_request": null,
+ "adjudications_written_on": "2026-10-09"
+ },
+ "2026-10-10": {
+ "commit": null,
+ "release": "dashboard-data-20261010",
"pull_request": null,
- "adjudications_written_on": "2026-10-06"
+ "adjudications_written_on": "2026-10-10"
}
},
"cases": {
@@ -568,8 +580,8 @@
"case_failure_source": "llm_error",
"case_failure_subtype": "thresholds_rates",
"reference_suspect": false,
- "judged_at_utc": "2026-09-29T01:49:51.816587+00:00",
- "verdict_sha256": "debe0ac227595d88e7b2ec74b9995e165182c239dd5c124d073a1e722bb6c2a5"
+ "judged_at_utc": "2026-10-09T11:11:08.050372+00:00",
+ "verdict_sha256": "524f744c91fb09b511e995137a30465794edfd6c09e08517478057c5cff5ec50"
},
"previous": {
"tree": "20260922c",
@@ -579,14 +591,6 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-23T12:00:01.365298+00:00",
"verdict_sha256": "f6be4503a154edd7f8a1ab2ee95fa8c23b9affae7582d0b98c2d9774990218a8"
- },
- "published": {
- "release": "dashboard-data-20260922",
- "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5",
- "judge_model": "claude-opus-5-5",
- "case_failure_source": "llm_error",
- "case_failure_subtype": "thresholds_rates",
- "judged_on_utc": null
}
},
"us__scenario_052__federal_income_tax_before_refundable_credits": {
@@ -1833,6 +1837,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-30T17:25:44.597220+00:00",
"verdict_sha256": "60720f64eb01fba3f686a353daa954df2f1495bd1e57364ca04288941d8613f6"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "judged_on_utc": "2026-09-30"
}
},
"us__scenario_032__payroll_tax": {
@@ -1844,6 +1856,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-29T01:49:48.761387+00:00",
"verdict_sha256": "354482e612feabf4d21b5c8d14b96f6d0d4fb34895d8cd83f183e8caaa20a862"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "judged_on_utc": "2026-09-29"
}
},
"us__scenario_043__payroll_tax": {
@@ -1855,6 +1875,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-22T18:14:01.503603+00:00",
"verdict_sha256": "3baf4b01b10a2d8e12915859767e1f9e366c0f1d59fe6cc7071ce3952431794e"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "judged_on_utc": "2026-09-22"
}
},
"us__scenario_081__federal_income_tax_before_refundable_credits": {
@@ -1866,6 +1894,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-30T04:09:19.928030+00:00",
"verdict_sha256": "6b046acbaf2b36276da1ea1f7fdb5b3cabe075ae69e156df9594f7c63f44cdb7"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "judged_on_utc": "2026-09-30"
}
},
"us__scenario_081__payroll_tax": {
@@ -1877,6 +1913,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-29T01:49:58.450053+00:00",
"verdict_sha256": "b29fd367d87b47db08504f5116ad0dfa315fb8b570074126b2689b4bef9fc54b"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "judged_on_utc": "2026-09-29"
}
},
"us__scenario_082__payroll_tax": {
@@ -1888,6 +1932,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-29T11:53:15.472783+00:00",
"verdict_sha256": "b7a8964530c183bb40079292b65272d9387a758efdd583b53e239096e06858fb"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "state_local_rule",
+ "judged_on_utc": "2026-09-29"
}
},
"us__scenario_114__federal_income_tax_before_refundable_credits": {
@@ -1899,6 +1951,14 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-30T04:34:26.960453+00:00",
"verdict_sha256": "ba069da56eb141ae8fa08c0acd7b72db4018c9d59f2dc525fe3e2eae87e2f6b6"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "judged_on_utc": "2026-09-30"
}
},
"us__scenario_114__state_income_tax_before_refundable_credits": {
@@ -1910,6 +1970,91 @@
"reference_suspect": false,
"judged_at_utc": "2026-09-30T04:35:28.792780+00:00",
"verdict_sha256": "4f07d313983947636f0243f0d1c4dee09517e8f783112c64b898557ff55ecec4"
+ },
+ "published": {
+ "release": "dashboard-data-20261006",
+ "commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "taxable_income_or_deductions",
+ "judged_on_utc": "2026-09-30"
+ }
+ },
+ "us__scenario_077__state_income_tax_before_refundable_credits": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "thresholds_rates",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:21:03.164972+00:00",
+ "verdict_sha256": "ac25f9a957e6df1a804ffbc0826fb5f27581670aedfe4512f1b136e2654f5e6f"
+ }
+ },
+ "us__scenario_093__federal_income_tax_before_refundable_credits": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:24:55.980556+00:00",
+ "verdict_sha256": "64c3a1b7c1e9dba12bd5460cf18033a610f4f11ed3c685a1977986ef3342c086"
+ }
+ },
+ "us__scenario_093__state_income_tax_before_refundable_credits": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:27:21.537119+00:00",
+ "verdict_sha256": "3bb92b597392168d3797344d6d533f5469324efe8f9bbcc246eed79e132a000e"
+ }
+ },
+ "us__scenario_123__federal_income_tax_before_refundable_credits": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:30:57.145495+00:00",
+ "verdict_sha256": "4f6f31d6328752b8321dd1bf304c4fc915568398d887aef734e362c41ac99e02"
+ }
+ },
+ "us__scenario_123__state_income_tax_before_refundable_credits": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "llm_error",
+ "case_failure_subtype": "household_unit_or_filing_status",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-09T11:32:09.911176+00:00",
+ "verdict_sha256": "ad43142d632889e7c01321c24189a6c1c30fac9148f5d24c28123cb66fa7b12e"
+ }
+ },
+ "us__scenario_015__local_income_tax": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-10T05:58:15.427599+00:00",
+ "verdict_sha256": "e2f324f633297e6cfcd3832de8cba9d5a71b8f022d3bb0694dd30a0cb6e9ce91"
+ }
+ },
+ "us__scenario_067__local_income_tax": {
+ "current": {
+ "tree": "stage",
+ "judge_model": "claude-opus-5-5",
+ "case_failure_source": "prompt_ambiguity",
+ "case_failure_subtype": "state_local_rule",
+ "reference_suspect": false,
+ "judged_at_utc": "2026-10-10T06:05:07.402541+00:00",
+ "verdict_sha256": "14c23bf7f38ae94b31c2f01fda8774837e4b599dcd43f38e2d21025c3cf19531"
}
}
}
diff --git a/reference_audit/2026-10-05-louisiana/proposed_exclusions.json b/reference_audit/2026-10-05-louisiana/proposed_exclusions.json
new file mode 100644
index 00000000..c21f6eb2
--- /dev/null
+++ b/reference_audit/2026-10-05-louisiana/proposed_exclusions.json
@@ -0,0 +1,38 @@
+{
+ "schema_version": 1,
+ "note": "The exclusion records for Max's ruling d994 (2026-10-06): exclude scenario_051 and scenario_077 state income tax before refundable credits, whose references rest on a 2026 Louisiana standard deduction ($12,835) that no Louisiana publication stated before the 2026-07-03 freeze; keep the published-amounts convention for every other reference. The release after dashboard-data-20261006 installs them (docs/haiku55/spec.json).",
+ "exclusions": [
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_law_published_after_freeze",
+ "alternative_reading": "R.S. 47:294(B) multiplies 2025's $12,500 by the calendar-2025 CPI-U increase but sets no rounding rule and gives the Department no duty to publish the result. Before the 2026-07-03 freeze Louisiana published only its withholding and estimated-tax amount, $12,875, and said the amount allowed on 2026 returns would differ; its first 2026 return amount, $12,838, came in RIB 26-019 on 2026-09-28, after the freeze. policyengine-us carries $12,835, which PolicyEngine/policyengine-us#8411 computed on 2026-05-24 from published CPI-U values ($12,500 x 324.054 / 315.605, rounded to the dollar): a computation under the statute, not a forecast, but a figure no Louisiana publication stated before the freeze. The reference is 3% of this single filer's Louisiana AGI ($40,180) less $12,835. Both Louisiana outputs that rest on the amount are excluded, and the published-amounts convention stays in force for every other reference (decision d994).",
+ "frozen_value": 820.3499755859375,
+ "alternative_value": 820.260009765625,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d994",
+ "root_cause": "la_2026_standard_deduction_unpublished",
+ "published": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "law": "La. R.S. 47:294(A)-(B) (Act 11 of the 2024 Third Extraordinary Session); BLS CPI-U, series CUUR0000SA0, December 2024 and December 2025 (released 2026-01-13); Louisiana Department of Revenue, Revenue Information Bulletin 26-019 (2026-09-28)",
+ "note": "Found 2026-10-05 while preparing an interview on the board, and audited in reference_audit/2026-10-05-louisiana (PolicyEngine/policybench#192): the sweep (scripts/sweep_la_standard_deduction.py, policyengine-us 2.15.17 with latest_final) reproduces all 1,928 scored references at $12,835 and moves only these two outputs under every candidate amount (verification/sweep_la_standard_deduction.log). Values under the candidates: $12,500 (2025's amount, the last return amount published) $830.40; $12,875 (the withholding amount) $819.15; $12,838 (RIB 26-019) $820.26. Of release dashboard-data-20260930's 46 models, none answered within $1 of the frozen value or of $820.26, and 21 answered within $1 of $830.40 (verification/leaderboard_impact.json)."
+ },
+ {
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason_code": "reference_law_published_after_freeze",
+ "alternative_reading": "R.S. 47:294(B) multiplies 2025's $12,500 by the calendar-2025 CPI-U increase but sets no rounding rule and gives the Department no duty to publish the result. Before the 2026-07-03 freeze Louisiana published only its withholding and estimated-tax amount, $12,875, and said the amount allowed on 2026 returns would differ; its first 2026 return amount, $12,838, came in RIB 26-019 on 2026-09-28, after the freeze. policyengine-us carries $12,835, which PolicyEngine/policyengine-us#8411 computed on 2026-05-24 from published CPI-U values ($12,500 x 324.054 / 315.605, rounded to the dollar): a computation under the statute, not a forecast, but a figure no Louisiana publication stated before the freeze. The reference is 3% of this single filer's Louisiana AGI ($23,006.18) less $12,835. Both Louisiana outputs that rest on the amount are excluded, and the published-amounts convention stays in force for every other reference (decision d994).",
+ "frozen_value": 305.13525390625,
+ "alternative_value": 305.0452575683594,
+ "engine_version": "policyengine-us 2.15.17",
+ "decided_on": "2026-10-06",
+ "decided_by": "developer",
+ "decision": "d994",
+ "root_cause": "la_2026_standard_deduction_unpublished",
+ "published": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
+ "law": "La. R.S. 47:294(A)-(B) (Act 11 of the 2024 Third Extraordinary Session); BLS CPI-U, series CUUR0000SA0, December 2024 and December 2025 (released 2026-01-13); Louisiana Department of Revenue, Revenue Information Bulletin 26-019 (2026-09-28)",
+ "note": "Found 2026-10-05 while preparing an interview on the board, and audited in reference_audit/2026-10-05-louisiana (PolicyEngine/policybench#192): the sweep (scripts/sweep_la_standard_deduction.py, policyengine-us 2.15.17 with latest_final) reproduces all 1,928 scored references at $12,835 and moves only these two outputs under every candidate amount (verification/sweep_la_standard_deduction.log). Values under the candidates: $12,500 (2025's amount, the last return amount published) $315.19; $12,875 (the withholding amount) $303.94; $12,838 (RIB 26-019) $305.05. Of release dashboard-data-20260930's 46 models, none answered within $1 of the frozen value or of $305.05, and 19 answered within $1 of $315.19 (verification/leaderboard_impact.json)."
+ }
+ ]
+}
diff --git a/reference_audit/2026-10-09-engine-upgrade/README.md b/reference_audit/2026-10-09-engine-upgrade/README.md
new file mode 100644
index 00000000..419f768a
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/README.md
@@ -0,0 +1,37 @@
+# Reference upgrade, October 10, 2026
+
+This directory records how PolicyBench's US references moved from policyengine-us 2.15.17 to
+policyengine-us 2.38.6. That was the first PyPI release carrying policyengine-us #10027, #10031,
+#10032 and #10034, and the newest release when PolicyBench began sweeping the references on
+2026-10-10 (`verification/sweep_timing.json`). The directory is named for the day the audit began
+(2026-10-09); the records date the upgrade by its UTC day, 2026-10-10.
+
+Of the 1,984 outputs:
+- 18 outputs are regenerated because 2.38.6 fixes the engine defects behind them. Release
+ 20261006 excluded 14 of them, which return to scoring. The other four were ruled on 2026-10-06
+ while still scored, and they stay scored.
+- 1 scored reference changes beyond the $1 tolerance: Idaho's scenario_076 state income tax, by
+ the $10 permanent building fund tax.
+- 2 outputs are newly excluded: the local income tax of Indiana's scenario_015 and scenario_067.
+ 2.38.6 computes it for a county no prompt states.
+- 2 scored references change by less than the $1 tolerance (scenario_082 state income tax,
+ scenario_085 federal income tax).
+- 17 excluded outputs whose values move were re-reviewed, and they stay excluded.
+
+## Files
+
+- `final_actions.json`: the reviewed actions the build ran on. Its sha256, `ef23b50e…`, is the
+ sidecar's `provenance.actions_sha256`.
+- `scripts/build_references_upgrade.py`: the builder. `scripts/sweep_timing.py`: the timing
+ record and the publication check (`verification/sweep_timing.json`).
+- `upstreams.json`: the upstream fix each regeneration names.
+- `evidence/`: the fix-module values on policyengine-us 2.37.2 and the upstream bisect.
+- `hand_corrected_narratives.json` and `.md`: the reference narratives of the changed outputs.
+
+## Erratum
+
+`final_actions.json`'s `note` was written before CA scenario_099's state income tax moved from the
+regenerations to the rechecks (commit a4e76adf). The note still says 19 regenerations and 16
+rechecks, but the file's arrays hold 18 regenerations and 17 rechecks. Those are the counts the
+sidecar, the paper and the benchmark card give. The note stays as written because the sidecar
+pins the file's bytes (`tests/test_reference_upgrade.py`).
diff --git a/reference_audit/2026-10-09-engine-upgrade/actions_from_cells.py b/reference_audit/2026-10-09-engine-upgrade/actions_from_cells.py
new file mode 100644
index 00000000..2ffaa711
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/actions_from_cells.py
@@ -0,0 +1,557 @@
+"""Draft an actions file for build_references_upgrade.py from the US hub's sweep.
+
+Input: the hub's cells_pe.csv (every output on the new release against the
+20261006 references: scenario_id, state, variable, frozen_20261006, new, delta,
+moved, status, audited_target), optionally the builder's computed.csv (or any
+sweep.py output) for full-precision values, the release spec
+(docs/haiku55/spec.json, through scripts/finish_haiku55.py) and the exclusion
+records (20261006's, read from git, and the release's ruled ten).
+
+Output: a DRAFT actions file ("draft": true, which the builder refuses until a
+reviewer clears it) and, on stdout, every judgment call it drafted. Nothing is
+decided silently: each moved output is classified by
+
+ status scored, excluded in 20261006, or ruled in the release;
+ audited target whether the new value lands within $1 of its record's
+ corrected value (alternative_value) or, with --evidence, of
+ the corrected value the record's fix modules give on a
+ pre-fix engine (a "fix_modules" target, which the builder
+ also checks the modules leave unchanged on the new engine);
+ and the hub's own column;
+ reason engine defect, unlisted input, or a ruled reason (later law,
+ household scope);
+
+and lands in exactly one place:
+
+ regenerated_exclusions an engine-defect record the new engine lands on
+ (PROPOSED: confirm the upstream fix and its release);
+ excluded_rechecked any other excluded output that moves, with a drafted
+ reason (PROPOSED: check the reason);
+ approved / new_exclusions a scored move a RULES entry recognizes, with drafted
+ cause and basis or a drafted record (PROPOSED: check every
+ sentence against the build's trace);
+ review only a scored move no rule recognizes (UNDECIDED: the builder
+ refuses it as an unreviewed move until a reviewer adds it).
+
+kept_exclusions_from_release lists every ruled output that is not regenerated.
+
+ PYTHONPATH= python actions_from_cells.py \\
+ --cells ~/reviews/policybench-pe-upgrade-2026-10-09/cells_pe2.37.1.csv \\
+ [--computed /computed.csv] [--date YYYY-MM-DD] \\
+ [--evidence reference_audit/2026-10-09-engine-upgrade/evidence/pe.json] \\
+ --out actions.draft.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import datetime
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE / "scripts"))
+
+from build_references_upgrade import ( # noqa: E402
+ ENGINE_DEFECT,
+ PREVIOUS_ENGINE,
+ ROOT,
+ TARGET_FIX_MODULES,
+ UNNAMED_UPSTREAM,
+ Refusal,
+ beyond,
+ key_of,
+ load_base,
+ load_evidence,
+ release_record,
+ sha256,
+)
+
+UNLISTED = "reference_depends_on_unlisted_input"
+CELL_TOL = 0.006 # the hub's table rounds to cents
+TARGET_TOL = 1.0
+
+
+@dataclass(frozen=True)
+class Cell:
+ scenario_id: str
+ state: str
+ variable: str
+ previous: float
+ new: float
+ status: str
+ audited_target: str
+
+ @property
+ def key(self) -> tuple[str, str]:
+ return (self.scenario_id, self.variable)
+
+
+def money(value: float) -> str:
+ return f"${value:,.2f}"
+
+
+# --- Rules for scored moves -------------------------------------------------
+# Each rule recognizes one documented kind of scored move and drafts its action.
+# The text states what the hub documented (README.md, HUB_BRIEF.md) and what the
+# 2.36.0 rehearsal's traces show (local_income_tax = in_county_tax with county_str
+# ADAMS_COUNTY_IN for scenario_015 and 067; state_income_tax_before_refundable_credits
+# = id_income_tax_before_refundable_credits + id_pbf $10 for scenario_076); a
+# reviewer checks it against the build's trace.
+
+
+def _indiana_county(cell: Cell, engine: str, date: str) -> dict:
+ return {
+ "action": "new_exclusions",
+ "rule": "indiana_county_unlisted",
+ "entry": {
+ "scenario_id": cell.scenario_id,
+ "variable": cell.variable,
+ "reason_code": UNLISTED,
+ "unlisted_input": "county of residence (Indiana county income tax)",
+ "alternative_reading": (
+ "The prompt places the household in Indiana but states no county, "
+ "and Indiana's county income taxes (IC 6-3.6) differ by county. "
+ f"{engine} computes Indiana county tax (in_county_tax) in "
+ "local_income_tax at the rate of the county it assigns a household "
+ f"with no stated county (Adams County on {engine}). "
+ f"Read without a county, as {PREVIOUS_ENGINE} did, the household "
+ "owes no county tax."
+ ),
+ "frozen_value": cell.new,
+ "alternative_value": cell.previous,
+ "engine_version": engine,
+ "decided_on": date,
+ "decided_by": "developer",
+ "note": (
+ f"Found in the {date} engine upgrade (the US hub's sweep, "
+ "cells_pe.md: local-tax standardization wires Indiana county "
+ "income tax into local_income_tax). The alternative is the "
+ f"{PREVIOUS_ENGINE} reference."
+ ),
+ },
+ "call": (
+ "NEW EXCLUSION (unlisted input: county) - confirm the county the engine "
+ "assumes in the build trace and the alternative value"
+ ),
+ }
+
+
+def _idaho_building_fund(cell: Cell, engine: str, date: str) -> dict:
+ return {
+ "action": "approved",
+ "rule": "idaho_permanent_building_fund",
+ "entry": {
+ "scenario_id": cell.scenario_id,
+ "variable": cell.variable,
+ "value": cell.new,
+ "cause": "id_permanent_building_fund_tax",
+ "basis": (
+ "Idaho's $10 permanent building fund tax counts in state income "
+ f"tax: on {engine}, state_income_tax_before_refundable_credits adds "
+ "id_pbf ($10 for a filer liable for it, id_pbf_liable; Idaho Form 40 "
+ "instructions) to Idaho income tax before "
+ "refundable credits, since policyengine-us 2.30.0 "
+ "(PolicyEngine/policyengine-us#9810)."
+ ),
+ },
+ "call": (
+ "APPROVE (+$10 permanent building fund tax, id_pbf in the 2.36.0 "
+ "trace) - confirm the trace and the Idaho Code citation"
+ ),
+ }
+
+
+RULES = (
+ (
+ lambda c: (
+ c.state == "IN"
+ and c.variable == "local_income_tax"
+ and c.previous == 0
+ and c.new > 0
+ ),
+ _indiana_county,
+ ),
+ (
+ lambda c: (
+ c.state == "ID"
+ and c.variable == "state_income_tax_before_refundable_credits"
+ and abs(c.new - c.previous - 10) <= CELL_TOL
+ ),
+ _idaho_building_fund,
+ ),
+)
+
+
+# --- Inputs --------------------------------------------------------------------
+
+
+def read_cells(path: Path, board: dict, computed: dict | None) -> list[Cell]:
+ with Path(path).open(newline="") as source:
+ rows = list(csv.DictReader(source))
+ keys = {(r["scenario_id"], r["variable"]) for r in rows}
+ if keys != set(board) or len(rows) != len(board):
+ raise Refusal("the cells table does not list exactly the 20261006 outputs")
+ cells = []
+ for r in rows:
+ k = (r["scenario_id"], r["variable"])
+ if abs(float(r["frozen_20261006"]) - board[k]) > CELL_TOL:
+ raise Refusal(f"{k}: the table's frozen value is not 20261006's")
+ new = float(r["new"])
+ if computed is not None:
+ if abs(computed[k] - new) > CELL_TOL:
+ raise Refusal(f"{k}: computed {computed[k]} is not the table's {new}")
+ new = computed[k]
+ cells.append(
+ Cell(
+ k[0],
+ r["state"],
+ k[1],
+ board[k],
+ new,
+ r["status"],
+ r.get("audited_target", ""),
+ )
+ )
+ return cells
+
+
+def read_computed(path: Path) -> dict:
+ with Path(path).open(newline="") as source:
+ return {
+ (r["scenario_id"], r["variable"]): float(r["recomputed"])
+ for r in csv.DictReader(source)
+ }
+
+
+def engine_from_path(path: Path) -> str:
+ match = re.fullmatch(r"cells_pe(\d+\.\d+\.\d+)\.csv", Path(path).name)
+ if not match:
+ raise Refusal(f"cannot read the engine from {Path(path).name}; pass --engine")
+ return match.group(1)
+
+
+# --- Drafting ------------------------------------------------------------------
+
+
+def first_upgrade_rechecks(meta: dict) -> dict:
+ first = [r for r in meta["revisions"] if r["kind"] == "engine_upgrade"][0]
+ return {key_of(r): r for r in first["excluded_outputs_rechecked"]}
+
+
+def second_reason(record: dict) -> str:
+ """The record's note when it names an unlisted input besides the defect."""
+ note = str(record.get("note", ""))
+ return note if "unlisted input" in note.lower() else ""
+
+
+def record_basis(record: dict) -> str:
+ return str(
+ record.get("unlisted_input")
+ or record.get("root_cause")
+ or record["reason_code"]
+ )
+
+
+def upstream_for(record: dict, key, upstreams: dict | None) -> str:
+ """The upstream fix a regeneration names: the output's own entry in
+ ``upstreams``, else each of its root causes' (joined with "; "), else the
+ record's own upstream."""
+ if not upstreams:
+ return record.get("upstream", "")
+ own = upstreams.get("outputs", {}).get(f"{key[0]}|{key[1]}")
+ if own:
+ return own
+ causes = str(record.get("root_cause", "")).split("+")
+ fixes = [upstreams.get("root_causes", {}).get(cause) for cause in causes]
+ if causes and all(fixes):
+ return "; ".join(dict.fromkeys(fixes))
+ return record.get("upstream", "")
+
+
+def draft_actions(
+ cells: list[Cell],
+ release: dict,
+ ruled: set,
+ base_excluded: set,
+ first_rechecks: dict,
+ *,
+ engine_version: str,
+ date: str,
+ spec_sha256: str,
+ precise: bool,
+ evidence: dict | None = None,
+ upstreams: dict | None = None,
+) -> tuple[dict, list[dict]]:
+ """The draft actions file and the judgment calls, one per moved output.
+
+ ``evidence`` is {"ref": {path, sha256}, "doc": the loaded evidence file}:
+ a pre-fix engine's values for engine-defect outputs without and with
+ their fix modules."""
+ evidence_items = {}
+ if evidence is not None:
+ for item in evidence["doc"]["items"]:
+ evidence_items[key_of(item)] = item
+ engine = f"policyengine-us {engine_version}"
+ ver_key = "value_on_2_15_17"
+ records = {key_of(r): r for r in release["exclusions"]}
+ actions = {
+ "engine": engine,
+ "previous_engine": PREVIOUS_ENGINE,
+ "date": date,
+ "release_spec_sha256": spec_sha256,
+ "draft": True,
+ "approved": [],
+ "regenerated_exclusions": [],
+ "new_exclusions": [],
+ "excluded_rechecked": [],
+ "kept_exclusions_from_release": [],
+ }
+ calls = []
+ regenerated = set()
+
+ def call(cell: Cell, decision: str, text: str, where: str) -> None:
+ calls.append(
+ {
+ "scenario_id": cell.scenario_id,
+ "state": cell.state,
+ "variable": cell.variable,
+ "previous": cell.previous,
+ "new": cell.new,
+ "status": cell.status,
+ "hub_audited_target": cell.audited_target,
+ "decision": decision,
+ "placed_in": where,
+ "call": text,
+ }
+ )
+
+ for cell in cells:
+ k = cell.key
+ if not beyond(cell.variable, cell.previous, cell.new):
+ continue
+ if k in records:
+ record = records[k]
+ source = "20261006" if k in base_excluded else "the release's ruling"
+ kept_value = cell.previous
+ if record["reason_code"] == ENGINE_DEFECT:
+
+ def lands(aim: float) -> bool:
+ return abs(cell.new - aim) <= TARGET_TOL and not beyond(
+ cell.variable, aim, cell.new
+ )
+
+ target = float(record["alternative_value"])
+ target_spec = {"kind": "record"}
+ target_text = f"the corrected value {money(target)} the "
+ target_text += f"{record['decided_on']} record computed"
+ item = evidence_items.get(k)
+ if not lands(target) and item is not None:
+ target = float(item["corrected_value"])
+ target_spec = {
+ "kind": TARGET_FIX_MODULES,
+ "modules": list(item["modules"]),
+ "evidence": dict(evidence["ref"]),
+ }
+ target_text = (
+ f"the corrected value {money(target)} its fix modules "
+ f"({' + '.join(item['modules'])}) give on "
+ f"{evidence['doc']['engine']}, which still has the defect "
+ f"({money(float(item['engine_value']))} without them)"
+ )
+ on_target = lands(target)
+ second = second_reason(record)
+ if on_target and second:
+ reason = (
+ f"Stays excluded on {engine}: the engine gives "
+ f"{money(cell.new)}, on the record's corrected value "
+ f"{money(target)}, but the record names a second reason "
+ f"that a new engine does not settle ({second}); the record "
+ f"keeps {money(kept_value)}."
+ )
+ actions["excluded_rechecked"].append(
+ {"scenario_id": k[0], "variable": k[1], "reason": reason}
+ )
+ call(
+ cell,
+ "PROPOSED",
+ f"RECHECK ({source} engine defect {record['root_cause']} "
+ "lands on target, but the record's note names an unlisted "
+ "input too): regenerate only if a reviewer rules that "
+ "reason out",
+ "excluded_rechecked",
+ )
+ continue
+ if on_target:
+ regenerated.add(k)
+ upstream = upstream_for(record, k, upstreams)
+ actions["regenerated_exclusions"].append(
+ {
+ "scenario_id": k[0],
+ "variable": k[1],
+ "alternative_value": float(record["alternative_value"]),
+ "tolerance": TARGET_TOL,
+ "upstream": upstream,
+ "basis": (
+ f"An engine defect fixed upstream ({upstream}) is "
+ f"regenerated with the fix: {engine} gives "
+ f"{money(cell.new)}, within $1 of {target_text} "
+ f"({record['root_cause']}: {record['defect']})."
+ ),
+ "target": target_spec,
+ }
+ )
+ hub = cell.audited_target
+ note = (
+ ""
+ if hub in ("", "yes")
+ else f" The hub's table disagrees ({hub}); resolve it first."
+ )
+ if k in first_rechecks:
+ note += (
+ " It was rechecked on 2026-09-29: read that reason in "
+ "the first engine_upgrade revision."
+ )
+ if upstream.strip().lower() in UNNAMED_UPSTREAM:
+ note += (
+ " The record names no upstream fix; the builder "
+ "refuses the regeneration until upstream names it."
+ )
+ call(
+ cell,
+ "PROPOSED",
+ f"REGENERATE ({source} engine defect {record['root_cause']}): "
+ f"lands on {money(cell.new)} vs target {money(target)}. "
+ "Confirm the fix is released upstream and name it in "
+ f"upstream ({upstream or 'empty'}).{note}",
+ "regenerated_exclusions",
+ )
+ continue
+ reason = (
+ f"Stays excluded on {engine}: the engine gives {money(cell.new)}, "
+ f"{money(abs(cell.new - target))} from the record's corrected "
+ f"value {money(target)}, so the defect ({record['root_cause']}) "
+ f"is not fixed in this release; the record keeps "
+ f"{money(kept_value)}."
+ )
+ if cell.audited_target.startswith("pending"):
+ reason += (
+ " The US hub's sweep marks it pending on more than one fix "
+ f"({cell.audited_target.removeprefix('pending: ')})."
+ )
+ text = (
+ f"RECHECK ({source} engine defect {record['root_cause']}, off "
+ f"target by {money(abs(cell.new - target))})"
+ )
+ else:
+ reason = (
+ f"Stays excluded on {engine}: the reference depends on "
+ f"{record_basis(record)} ({record['reason_code']}), which an "
+ f"engine release does not settle; the record keeps "
+ f"{money(kept_value)}, and {engine} gives {money(cell.new)}."
+ )
+ text = f"RECHECK ({source} {record['reason_code']})"
+ earlier = first_rechecks.get(k)
+ if earlier is not None and abs(float(earlier[ver_key]) - cell.new) <= 1e-3:
+ reason += (
+ " The value is 2.15.17's, rechecked on 2026-09-29 (the first "
+ "engine_upgrade revision's excluded_outputs_rechecked)."
+ )
+ actions["excluded_rechecked"].append(
+ {"scenario_id": k[0], "variable": k[1], "reason": reason}
+ )
+ call(
+ cell,
+ "PROPOSED",
+ text + " - check the drafted reason",
+ "excluded_rechecked",
+ )
+ continue
+ for matches, propose in RULES:
+ if matches(cell):
+ proposal = propose(cell, engine, date)
+ actions[proposal["action"]].append(proposal["entry"])
+ text = proposal["call"]
+ if not precise:
+ text += (
+ "; the value is the table's, rounded to cents: rerun with "
+ "--computed (the builder's computed.csv) before the build"
+ )
+ call(cell, "PROPOSED", text, proposal["action"])
+ break
+ else:
+ call(
+ cell,
+ "UNDECIDED",
+ "scored move no rule recognizes: approve it (cause, basis) or "
+ "exclude it (a complete record); the builder refuses it until then",
+ "review only",
+ )
+ actions["kept_exclusions_from_release"] = [
+ {"scenario_id": k[0], "variable": k[1]} for k in sorted(ruled - regenerated)
+ ]
+ for name in (
+ "approved",
+ "regenerated_exclusions",
+ "new_exclusions",
+ "excluded_rechecked",
+ ):
+ actions[name].sort(key=key_of)
+ actions["review"] = calls
+ return actions, calls
+
+
+def print_calls(calls: list[dict]) -> None:
+ print(f"{len(calls)} moved outputs; every call below needs a reviewer:")
+ for c in calls:
+ print(
+ f" [{c['decision']}] {c['scenario_id']} {c['state']} {c['variable']}: "
+ f"{c['previous']:,.2f} -> {c['new']:,.2f} ({c['status']}) -> "
+ f"{c['placed_in']}\n {c['call']}"
+ )
+
+
+def main(argv: list[str] | None = None) -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--cells", required=True)
+ parser.add_argument("--computed")
+ parser.add_argument("--engine")
+ parser.add_argument("--date")
+ parser.add_argument("--evidence", help="a committed evidence file, repo-relative")
+ parser.add_argument("--upstreams", help="upstreams.json: the fix each regeneration names")
+ parser.add_argument("--out", required=True)
+ args = parser.parse_args(argv)
+ evidence = None
+ if args.evidence:
+ ref = {"path": args.evidence, "sha256": sha256(ROOT / args.evidence)}
+ evidence = {"ref": ref, "doc": load_evidence(ref)}
+ engine_version = args.engine or engine_from_path(Path(args.cells))
+ date = args.date or datetime.datetime.now(datetime.timezone.utc).date().isoformat()
+ base = load_base()
+ release, ruled, spec_sha = release_record(base.exclusions)
+ computed = read_computed(Path(args.computed)) if args.computed else None
+ cells = read_cells(Path(args.cells), base.reference.values(), computed)
+ actions, calls = draft_actions(
+ cells,
+ release,
+ ruled,
+ {key_of(r) for r in base.exclusions["exclusions"]},
+ first_upgrade_rechecks(base.meta),
+ engine_version=engine_version,
+ date=date,
+ spec_sha256=spec_sha,
+ precise=computed is not None,
+ evidence=evidence,
+ upstreams=json.loads(Path(args.upstreams).read_text()) if args.upstreams else None,
+ )
+ Path(args.out).write_text(json.dumps(actions, indent=2) + "\n")
+ print_calls(calls)
+ print(f"DRAFT -> {args.out} (draft: true; the builder refuses it until reviewed)")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json b/reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json
new file mode 100644
index 00000000..eacbf586
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json
@@ -0,0 +1,248 @@
+{
+ "kind": "regeneration_evidence",
+ "engine": "policyengine-us 2.37.2",
+ "policyengine_core": "3.32.29",
+ "computed_at_utc": "2026-10-09T18:04:05.758398+00:00",
+ "builder_sha256": "7abe3a5919db195f5932bb569421fedeb718da059284b5e7a98022e4168e35e4",
+ "harness_sha256": "f4c7783453a473a1bedcf64ef554c3a2fc29f9d663626948d920ed3ef2ea2b69",
+ "conventions": "reference_audit/2026-09-28/fixes/latest_final.py",
+ "base_commit": "9ce4ade8382962a9134860c23f56d92509b5e57f",
+ "modules": {
+ "r01_ira_compensation.py": "167d1d549e760765ccfe89e11beaa0131e6fc67ab93f4c69683ac0b0e6877ea1",
+ "r01_ira_compensation_v2.py": "652328ea94e967b7cd36e990d5064fa0709767be431db1f2f0fe6cdcb2a63050",
+ "r02_ira_219g.py": "295a0b975ced8964fc1b2c5d52c47bce177eed399f49735eafb88441e7561b84",
+ "r02_ira_219g_v2.py": "937514fb1199c4ca172dfc5252db7f2483adade193922e6e5283d0b6ab4c4c8c",
+ "r03_estate_income__qbi_false_v2.py": "86dbaf59a22c0029e608167d10d38994c507a5e1cc9293592366ad3cdf1eb280",
+ "r03_estate_income_v2.py": "e0620ea2692ed1711fb2791649db1678b75c050efc904965ea4e5e2d9883bdbb",
+ "r05_nj_worker_ui_v2.py": "66322c7c4d0bc480e0e857d1e176a387499b26ac71f18788415345f6fc8d0dda",
+ "r06_wi_act15_before_refundable_v2.py": "1128ee23524a87bf53facee969833f99e28b731534381732773d801d71e29ada",
+ "r07_idaho_health_premiums_v2.py": "af9b7cc6dbd25b1e60b547cc1bee167527dbf7faee4dc88512c4c28416808222",
+ "r08_eitc_earned_income_deferrals_v2.py": "ba57e9e15f4a48a9e9aa4a916ed067246ecee28df64fc76f4633da044cf8bd9b",
+ "r11_ca_itemized_conformity_v2.py": "2bcb8ba6694c053f3c9a8509b96bfee4976ba00e3fb676c0c504e06c2d6c046d",
+ "r30_snap_heat_and_eat_sua.py": "11072570d931a4e77e50216e4f7c926c83749e3d437afaa8f34442d102cd6b82",
+ "r32_wi_capital_gain_distributions.py": "1b25c10c4a6139e0b9e2a1a6eb764932c91bb3036d5ff69986d7d6469c10f823"
+ },
+ "items": [
+ {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r01_ira_compensation_v2.py"
+ ],
+ "engine_value": 22154.69921875,
+ "corrected_value": 22400.654296875
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 107833.15625,
+ "corrected_value": 108525.578125
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r11_ca_itemized_conformity_v2.py"
+ ],
+ "engine_value": 41267.01171875,
+ "corrected_value": 40975.25390625
+ },
+ {
+ "scenario_id": "scenario_007",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r07_idaho_health_premiums_v2.py"
+ ],
+ "engine_value": 771.5570068359375,
+ "corrected_value": 661.3170166015625
+ },
+ {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "modules": [
+ "r05_nj_worker_ui_v2.py"
+ ],
+ "engine_value": 2162.760009765625,
+ "corrected_value": 2276.659912109375
+ },
+ {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r11_ca_itemized_conformity_v2.py"
+ ],
+ "engine_value": 2505.8701171875,
+ "corrected_value": 1968.805419921875
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r03_estate_income__qbi_false_v2.py"
+ ],
+ "engine_value": 7729.669921875,
+ "corrected_value": 8596.0283203125
+ },
+ {
+ "scenario_id": "scenario_042",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r06_wi_act15_before_refundable_v2.py",
+ "r32_wi_capital_gain_distributions.py"
+ ],
+ "engine_value": 464.18280029296875,
+ "corrected_value": 0.0
+ },
+ {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 30543.908203125,
+ "corrected_value": 30702.58984375
+ },
+ {
+ "scenario_id": "scenario_052",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 104211.40625,
+ "corrected_value": 104225.2578125
+ },
+ {
+ "scenario_id": "scenario_053",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r07_idaho_health_premiums_v2.py"
+ ],
+ "engine_value": 2451.05712890625,
+ "corrected_value": 2186.05712890625
+ },
+ {
+ "scenario_id": "scenario_080",
+ "variable": "snap",
+ "modules": [
+ "r30_snap_heat_and_eat_sua.py"
+ ],
+ "engine_value": 3576.0,
+ "corrected_value": 3240.0
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 9577.90234375,
+ "corrected_value": 9579.767578125
+ },
+ {
+ "scenario_id": "scenario_091",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r32_wi_capital_gain_distributions.py"
+ ],
+ "engine_value": 895.8131103515625,
+ "corrected_value": 878.5157470703125
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 10679.75,
+ "corrected_value": 10711.484375
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r11_ca_itemized_conformity_v2.py"
+ ],
+ "engine_value": 4640.77734375,
+ "corrected_value": 4588.48388671875
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "federal_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ],
+ "engine_value": 2878.0966796875,
+ "corrected_value": 822.2882690429688
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "state_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ],
+ "engine_value": 473.1777038574219,
+ "corrected_value": 164.45765686035156
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r03_estate_income__qbi_false_v2.py"
+ ],
+ "engine_value": 25396.6640625,
+ "corrected_value": 25747.7734375
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r03_estate_income__qbi_false_v2.py"
+ ],
+ "engine_value": 4231.35302734375,
+ "corrected_value": 4236.8076171875
+ },
+ {
+ "scenario_id": "scenario_119",
+ "variable": "federal_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ],
+ "engine_value": 2854.0087890625,
+ "corrected_value": 3366.07373046875
+ },
+ {
+ "scenario_id": "scenario_119",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ],
+ "engine_value": 1859.0943603515625,
+ "corrected_value": 1756.681396484375
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 40416.66796875,
+ "corrected_value": 40423.17578125
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "engine_value": 11917.18359375,
+ "corrected_value": 11919.0576171875
+ }
+ ]
+}
diff --git a/reference_audit/2026-10-09-engine-upgrade/evidence/request.json b/reference_audit/2026-10-09-engine-upgrade/evidence/request.json
new file mode 100644
index 00000000..434094f4
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/evidence/request.json
@@ -0,0 +1,177 @@
+{
+ "items": [
+ {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r01_ira_compensation_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r11_ca_itemized_conformity_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_007",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r07_idaho_health_premiums_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "modules": [
+ "r05_nj_worker_ui_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r11_ca_itemized_conformity_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r03_estate_income__qbi_false_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_042",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r06_wi_act15_before_refundable_v2.py",
+ "r32_wi_capital_gain_distributions.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_052",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_053",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r07_idaho_health_premiums_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_080",
+ "variable": "snap",
+ "modules": [
+ "r30_snap_heat_and_eat_sua.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_091",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r32_wi_capital_gain_distributions.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r11_ca_itemized_conformity_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "federal_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "state_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r03_estate_income__qbi_false_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r03_estate_income__qbi_false_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_119",
+ "variable": "federal_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_119",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r08_eitc_earned_income_deferrals_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/reference_audit/2026-10-09-engine-upgrade/evidence/upstream_bisect.md b/reference_audit/2026-10-09-engine-upgrade/evidence/upstream_bisect.md
new file mode 100644
index 00000000..3b118ddd
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/evidence/upstream_bisect.md
@@ -0,0 +1,21 @@
+# Upstream fixes behind two "record" regenerations
+
+Two release 20261006 engine-defect records land on their own audited corrected
+value (alternative_value) on policyengine-us 2.37.2. Nothing else in the engine
+has moved them since the records were decided. To name the upstream fix each
+regeneration cites, `scripts/probe_cells.py` computed each output with the
+pinned conventions (`reference_audit/2026-09-28/fixes/latest_final.py`). It ran
+on a checkout of the candidate pull request's merge commit and on its first
+parent. Both checkouts used the policyengine-core 3.32.29 environment of the
+2.37.2 evidence, run on 2026-10-09.
+
+| Output | Record's corrected value | Parent | Merge | Fix |
+|---|---:|---:|---:|---|
+| scenario_064 federal_income_tax_before_refundable_credits (r01_ira_compensation) | 4,441.455078 | 4,439.289551 | 4,441.455078 | PolicyEngine/policyengine-us#9801 (merge 1f9d85aa, parent 4732913a), "Keep dependents' above-the-line deductions off the filer's AGI" |
+| scenario_064 state_income_tax_before_refundable_credits (r01_ira_compensation) | 4,599.621582 | 4,598.476074 | 4,599.621582 | #9801 |
+| scenario_039 state_income_tax_before_refundable_credits (r03_estate_income) | 1,975.798218 | 514.498413 | 1,975.798218 | PolicyEngine/policyengine-us#9633 (merge e77257c4, parent 7968789c), "Include estate and trust income in federal gross income and the NIIT base" |
+
+On the merge commit, each output equals its record's corrected value to the
+cent. On the parent it does not. scenario_039's federal output, the record's
+other half (estate income counted as qualified business income), waits on
+PolicyEngine/policyengine-us#10027.
diff --git a/reference_audit/2026-10-09-engine-upgrade/final_actions.json b/reference_audit/2026-10-09-engine-upgrade/final_actions.json
new file mode 100644
index 00000000..40d6c55c
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/final_actions.json
@@ -0,0 +1,831 @@
+{
+ "engine": "policyengine-us 2.38.6",
+ "previous_engine": "policyengine-us 2.15.17",
+ "date": "2026-10-10",
+ "release_spec_sha256": "5e7856a42548746bc37760adea1af718050429e0b9c87c4d62755c7d036dab71",
+ "draft": false,
+ "approved": [
+ {
+ "scenario_id": "scenario_076",
+ "variable": "state_income_tax_before_refundable_credits",
+ "value": 6828.34423828125,
+ "cause": "id_permanent_building_fund_tax",
+ "basis": "Idaho's $10 permanent building fund tax counts in state income tax: on policyengine-us 2.38.6, state_income_tax_before_refundable_credits adds id_pbf ($10 for a filer liable for it, id_pbf_liable; Idaho Form 40 instructions) to Idaho income tax before refundable credits, since policyengine-us 2.30.0 (PolicyEngine/policyengine-us#9810)."
+ }
+ ],
+ "regenerated_exclusions": [
+ {
+ "scenario_id": "scenario_003",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 22400.654297,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $22,400.65, within $1 of the corrected value $22,400.65 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_005",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 40920.402344,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $40,975.25, within $1 of the corrected value $40,975.25 its fix modules (r02_ira_219g_v2.py + r11_ca_itemized_conformity_v2.py) give on policyengine-us 2.37.2, which still has the defect ($41,267.01 without them) (r02_ira_219g+r11_ca_itemized_conformity: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions).",
+ "target": {
+ "kind": "fix_modules",
+ "modules": [
+ "r02_ira_219g_v2.py",
+ "r11_ca_itemized_conformity_v2.py"
+ ],
+ "evidence": {
+ "path": "reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json",
+ "sha256": "4ccc7160b0d8513dedab1fb60ef57bfbd08cb495f445e7ca955141cb6c6c17a6"
+ }
+ }
+ },
+ {
+ "scenario_id": "scenario_008",
+ "variable": "payroll_tax",
+ "alternative_value": 2276.659912,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10031",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10031) is regenerated with the fix: policyengine-us 2.38.6 gives $2,276.66, within $1 of the corrected value $2,276.66 the 2026-09-22 record computed (r05_nj_worker_ui: PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_018",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 1137.302246,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9928",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9928) is regenerated with the fix: policyengine-us 2.38.6 gives $1,137.30, within $1 of the corrected value $1,137.30 the 2026-10-06 record computed (az_standard_deduction_indexing: policyengine-us puts the Arizona standard deduction's uprating on the parent parameter node, which policyengine-core does not propagate to the filing-status children, so the 2025 amount ($15,750 single) carries into 2026 unindexed).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_022",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 1968.80542,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10034",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10034) is regenerated with the fix: policyengine-us 2.38.6 gives $1,968.81, within $1 of the corrected value $1,968.81 the 2026-09-22 record computed (r11_ca_itemized_conformity: PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_025",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 1916.604,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925) is regenerated with the fix: policyengine-us 2.38.6 gives $1,916.61, within $1 of the corrected value $1,916.60 the 2026-10-06 record computed (oh_medical_deduction_premiums: policyengine-us counts health insurance premiums toward Ohio's 7.5%-of-federal-AGI medical deduction only when the person is Medicare-eligible AND has no employer-paid plan, where Ohio's worksheet line 3 reads 'Medicare or an employer-paid health care plan', so a non-Medicare filer with employer coverage gets no premium credit).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 8596.02832,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $8,596.03, within $1 of the corrected value $8,596.03 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_039",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 1975.798218,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9633",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9633) is regenerated with the fix: policyengine-us 2.38.6 gives $1,975.80, within $1 of the corrected value $1,975.80 the 2026-09-22 record computed (r03_estate_income: PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_043",
+ "variable": "state_refundable_credits",
+ "alternative_value": 0.0,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9946",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9946) is regenerated with the fix: policyengine-us 2.38.6 gives $0.00, within $1 of the corrected value $0.00 the 2026-10-06 record computed (co_sales_tax_refund_surplus: policyengine-us pays Colorado's six-tier state sales tax refund without the statute's condition that the fiscal year ending in the tax year had excess state revenues, carrying the tax year 2025 table into 2026).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_049",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 30702.589844,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $30,702.59, within $1 of the corrected value $30,702.59 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 4441.455078,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9801",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,441.46, within $1 of the corrected value $4,441.46 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_064",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 4599.621582,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9801",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9801) is regenerated with the fix: policyengine-us 2.38.6 gives $4,599.62, within $1 of the corrected value $4,599.62 the 2026-09-22 record computed (r01_ira_compensation: PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 9564.916992,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $9,579.77, within $1 of the corrected value $9,579.77 its fix modules (r02_ira_219g_v2.py) give on policyengine-us 2.37.2, which still has the defect ($9,577.90 without them) (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "target": {
+ "kind": "fix_modules",
+ "modules": [
+ "r02_ira_219g_v2.py"
+ ],
+ "evidence": {
+ "path": "reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json",
+ "sha256": "4ccc7160b0d8513dedab1fb60ef57bfbd08cb495f445e7ca955141cb6c6c17a6"
+ }
+ }
+ },
+ {
+ "scenario_id": "scenario_082",
+ "variable": "state_refundable_credits",
+ "alternative_value": 1187.611,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#9948",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#9948) is regenerated with the fix: policyengine-us 2.38.6 gives $1,187.55, within $1 of the corrected value $1,187.61 the 2026-10-06 record computed (ny_cdcc_606_c2: policyengine-us computes New York's 2026 child and dependent care credit with the pre-2026 formula (a percentage of the federal credit), which L.2026 ch. 59 limited to tax years beginning before 2026 and replaced with the refundable 606(c-2) credit on qualifying expenses).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 10711.484375,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $10,711.48, within $1 of the corrected value $10,711.48 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "alternative_value": 25747.773438,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $25,747.77, within $1 of the corrected value $25,747.77 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_110",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 4236.807617,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027) is regenerated with the fix: policyengine-us 2.38.6 gives $4,236.81, within $1 of the corrected value $4,236.81 the 2026-09-22 record computed (r02_ira_219g+r03_estate_income: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income).",
+ "target": {
+ "kind": "record"
+ }
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "state_income_tax_before_refundable_credits",
+ "alternative_value": 11919.057617,
+ "tolerance": 1.0,
+ "upstream": "PolicyEngine/policyengine-us#10032",
+ "basis": "An engine defect fixed upstream (PolicyEngine/policyengine-us#10032) is regenerated with the fix: policyengine-us 2.38.6 gives $11,919.06, within $1 of the corrected value $11,919.06 the 2026-09-22 record computed (r02_ira_219g: PolicyEngine deducts traditional IRA contributions without the active-participant phase-out).",
+ "target": {
+ "kind": "record"
+ }
+ }
+ ],
+ "new_exclusions": [
+ {
+ "scenario_id": "scenario_015",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "unlisted_input": "county of residence (Indiana county income tax)",
+ "alternative_reading": "The prompt places the household in Indiana but states no county, and Indiana's county income taxes (IC 6-3.6) differ by county. policyengine-us 2.38.6 computes Indiana county tax (in_county_tax) in local_income_tax at the rate of the county it assigns a household with no stated county (Adams County on policyengine-us 2.38.6). Read without a county, as policyengine-us 2.15.17 did, the household owes no county tax.",
+ "frozen_value": 670.5501098632812,
+ "alternative_value": 0.0,
+ "engine_version": "policyengine-us 2.38.6",
+ "decided_on": "2026-10-10",
+ "decided_by": "developer",
+ "note": "Found in the 2026-10-10 engine upgrade (the US hub's sweep, cells_pe.md: local-tax standardization wires Indiana county income tax into local_income_tax). The alternative is the policyengine-us 2.15.17 reference."
+ },
+ {
+ "scenario_id": "scenario_067",
+ "variable": "local_income_tax",
+ "reason_code": "reference_depends_on_unlisted_input",
+ "unlisted_input": "county of residence (Indiana county income tax)",
+ "alternative_reading": "The prompt places the household in Indiana but states no county, and Indiana's county income taxes (IC 6-3.6) differ by county. policyengine-us 2.38.6 computes Indiana county tax (in_county_tax) in local_income_tax at the rate of the county it assigns a household with no stated county (Adams County on policyengine-us 2.38.6). Read without a county, as policyengine-us 2.15.17 did, the household owes no county tax.",
+ "frozen_value": 1348.83203125,
+ "alternative_value": 0.0,
+ "engine_version": "policyengine-us 2.38.6",
+ "decided_on": "2026-10-10",
+ "decided_by": "developer",
+ "note": "Found in the 2026-10-10 engine upgrade (the US hub's sweep, cells_pe.md: local-tax standardization wires Indiana county income tax into local_income_tax). The alternative is the policyengine-us 2.15.17 reference."
+ }
+ ],
+ "excluded_rechecked": [
+ {
+ "scenario_id": "scenario_005",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $108,525.58, on the record's corrected value $108,525.58, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input).); the record keeps $106,505.90."
+ },
+ {
+ "scenario_id": "scenario_007",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $771.56, $110.24 from the record's corrected value $661.32, so the defect (r07_idaho_health_premiums) is not fixed in this release; the record keeps $755.78."
+ },
+ {
+ "scenario_id": "scenario_020",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $68,056.71, and policyengine-us 2.38.6 gives $68,334.73. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_023",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $461.34, and policyengine-us 2.38.6 gives $408.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_042",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $464.18, $464.18 from the record's corrected value $0.00, so the defect (r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions) is not fixed in this release; the record keeps $284.74. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_052",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $104,225.26, on the record's corrected value $104,225.26, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input).); the record keeps $104,211.41."
+ },
+ {
+ "scenario_id": "scenario_053",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $2,451.06, $265.00 from the record's corrected value $2,186.06, so the defect (r07_idaho_health_premiums) is not fixed in this release; the record keeps $2,435.28."
+ },
+ {
+ "scenario_id": "scenario_056",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $1,140.00, and policyengine-us 2.38.6 gives $95.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_057",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $2,669.22, and policyengine-us 2.38.6 gives $2,628.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_080",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $3,576.00, $336.00 from the record's corrected value $3,240.00, so the defect (r30_snap_heat_and_eat_sua) is not fixed in this release; the record keeps $3,596.04. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_081",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $8,232.90, on the record's corrected value $8,232.90, but the record names a second reason that a new engine does not settle (Corrected value applies ma_part_a_loss_offset with every publication convention and upstream fix. The output also moves under r23_ma_interest_source (unlisted input).); the record keeps $8,238.41."
+ },
+ {
+ "scenario_id": "scenario_091",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $895.81, $17.30 from the record's corrected value $878.52, so the defect (r32_wi_capital_gain_distributions) is not fixed in this release; the record keeps $843.66. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_099",
+ "variable": "state_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $4,588.48, on the record's audited corrected value, but the reference rests on a further engine defect the new engine does not fix. California does not conform to the federal above-the-line deduction for educator expenses (R&TC 17072; FTB 2025 Instructions for Schedule CA (540), Section C, line 11: 'California law does not conform to federal law regarding educator expenses'), so California AGI adds back the spouse's $337.50 educator expense; policyengine-us sets ca_agi to federal AGI ($159,729.30) without it, and the tax is about $26.90 lower than the law gives. Found by the release's Opus 5.5 judge (reference flag on the re-opened case, 2026-10-10) and confirmed against the FTB instructions; not yet fixed upstream."
+ },
+ {
+ "scenario_id": "scenario_100",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on meets_ssi_disability_criteria (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $8,625.89, and policyengine-us 2.38.6 gives $8,556.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_112",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on weekly_hours_worked_before_lsr (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $287.68, and policyengine-us 2.38.6 gives $0.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_118",
+ "variable": "snap",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the reference depends on whether the listed home mortgage interest is on the home the SNAP household occupies (reference_depends_on_unlisted_input), which an engine release does not settle; the record keeps $2,903.94, and policyengine-us 2.38.6 gives $2,868.00. The value is 2.15.17's, rechecked on 2026-09-29 (the first engine_upgrade revision's excluded_outputs_rechecked)."
+ },
+ {
+ "scenario_id": "scenario_120",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "reason": "Stays excluded on policyengine-us 2.38.6: the engine gives $40,423.18, on the record's corrected value $40,423.18, but the record names a second reason that a new engine does not settle (Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input). The output also moves under the state income tax withheld reading (unlisted input): the reference's SALT uses the Connecticut withholding estimate ($12,010.05); with the engine's own liability ($11,917.18) the 2.15.17 value is $40,438.95, and with no state income tax paid it is $43,619.93. With r02's IRA fix on 2.15.17 the value is $40,423.18 under the withholding estimate, $40,445.46 under the liability and $43,629.21 with no state income tax paid. policyengine-us 2.15.17 also leaves the household's Connecticut paid leave contributions ($827.98) out of SALT (Rev. Rul. 2025-4); with them the 2.15.17 value is $40,217.95.); the record keeps $40,021.82."
+ }
+ ],
+ "kept_exclusions_from_release": [
+ {
+ "scenario_id": "scenario_051",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_077",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "federal_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_093",
+ "variable": "state_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "federal_income_tax_before_refundable_credits"
+ },
+ {
+ "scenario_id": "scenario_123",
+ "variable": "state_income_tax_before_refundable_credits"
+ }
+ ],
+ "review": [
+ {
+ "scenario_id": "scenario_003",
+ "state": "TX",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 22154.69921875,
+ "new": 22400.654296875,
+ "status": "excluded 20261006: r01_ira_compensation",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r01_ira_compensation): lands on $22,400.65 vs target $22,400.65. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032)."
+ },
+ {
+ "scenario_id": "scenario_005",
+ "state": "CA",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 106505.8984375,
+ "new": 108525.578125,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r02_ira_219g lands on target, but the record's note names an unlisted input too): regenerate only if a reviewer rules that reason out"
+ },
+ {
+ "scenario_id": "scenario_005",
+ "state": "CA",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 41051.51171875,
+ "new": 40975.25390625,
+ "status": "excluded 20261006: r02_ira_219g+r11_ca_itemized_conformity",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g+r11_ca_itemized_conformity): lands on $40,975.25 vs target $40,975.25. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034). It was rechecked on 2026-09-29: read that reason in the first engine_upgrade revision."
+ },
+ {
+ "scenario_id": "scenario_007",
+ "state": "ID",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 755.7783203125,
+ "new": 771.5570068359375,
+ "status": "excluded 20261006: r07_idaho_health_premiums",
+ "hub_audited_target": "no (target 661.32)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r07_idaho_health_premiums, off target by $110.24) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_008",
+ "state": "NJ",
+ "variable": "payroll_tax",
+ "previous": 2162.760009765625,
+ "new": 2276.659912109375,
+ "status": "excluded 20261006: r05_nj_worker_ui",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r05_nj_worker_ui): lands on $2,276.66 vs target $2,276.66. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10031)."
+ },
+ {
+ "scenario_id": "scenario_015",
+ "state": "IN",
+ "variable": "local_income_tax",
+ "previous": 0.0,
+ "new": 670.5501098632812,
+ "status": "scored",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "new_exclusions",
+ "call": "NEW EXCLUSION (unlisted input: county) - confirm the county the engine assumes in the build trace and the alternative value"
+ },
+ {
+ "scenario_id": "scenario_018",
+ "state": "AZ",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 1146.05224609375,
+ "new": 1137.30224609375,
+ "status": "excluded in Haiku release (d994/d1022): az_standard_deduction_indexing",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (the release's ruling engine defect az_standard_deduction_indexing): lands on $1,137.30 vs target $1,137.30. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9928)."
+ },
+ {
+ "scenario_id": "scenario_020",
+ "state": "TX",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 68056.7109375,
+ "new": 68334.734375,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_022",
+ "state": "CA",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 2439.650146484375,
+ "new": 1968.805419921875,
+ "status": "excluded 20261006: r11_ca_itemized_conformity",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r11_ca_itemized_conformity): lands on $1,968.81 vs target $1,968.81. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10034). It was rechecked on 2026-09-29: read that reason in the first engine_upgrade revision."
+ },
+ {
+ "scenario_id": "scenario_023",
+ "state": "CA",
+ "variable": "snap",
+ "previous": 461.3397216796875,
+ "new": 408.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_025",
+ "state": "OH",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 1921.57275390625,
+ "new": 1916.606201171875,
+ "status": "excluded in Haiku release (d994/d1022): oh_medical_deduction_premiums",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (the release's ruling engine defect oh_medical_deduction_premiums): lands on $1,916.61 vs target $1,916.60. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10020; related: PolicyEngine/policyengine-us#9925)."
+ },
+ {
+ "scenario_id": "scenario_039",
+ "state": "VA",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 1345.510498046875,
+ "new": 8596.0283203125,
+ "status": "excluded 20261006: r03_estate_income",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r03_estate_income): lands on $8,596.03 vs target $8,596.03. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10027)."
+ },
+ {
+ "scenario_id": "scenario_039",
+ "state": "VA",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 514.4984130859375,
+ "new": 1975.7982177734375,
+ "status": "excluded 20261006: r03_estate_income",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r03_estate_income): lands on $1,975.80 vs target $1,975.80. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9633)."
+ },
+ {
+ "scenario_id": "scenario_042",
+ "state": "WI",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 284.74090576171875,
+ "new": 464.18280029296875,
+ "status": "excluded 20261006: r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions",
+ "hub_audited_target": "no (target 0.00)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions, off target by $464.18) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_043",
+ "state": "CO",
+ "variable": "state_refundable_credits",
+ "previous": 19.0,
+ "new": 0.0,
+ "status": "excluded in Haiku release (d994/d1022): co_sales_tax_refund_surplus",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (the release's ruling engine defect co_sales_tax_refund_surplus): lands on $0.00 vs target $0.00. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9946)."
+ },
+ {
+ "scenario_id": "scenario_049",
+ "state": "NH",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 30543.908203125,
+ "new": 30702.58984375,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g): lands on $30,702.59 vs target $30,702.59. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032)."
+ },
+ {
+ "scenario_id": "scenario_052",
+ "state": "TX",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 104211.40625,
+ "new": 104225.2578125,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r02_ira_219g lands on target, but the record's note names an unlisted input too): regenerate only if a reviewer rules that reason out"
+ },
+ {
+ "scenario_id": "scenario_053",
+ "state": "ID",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 2435.2783203125,
+ "new": 2451.05712890625,
+ "status": "excluded 20261006: r07_idaho_health_premiums",
+ "hub_audited_target": "no (target 2186.06)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r07_idaho_health_premiums, off target by $265.00) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_056",
+ "state": "NJ",
+ "variable": "snap",
+ "previous": 1140.0,
+ "new": 95.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_057",
+ "state": "LA",
+ "variable": "snap",
+ "previous": 2669.217041015625,
+ "new": 2628.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_064",
+ "state": "WI",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 4439.28955078125,
+ "new": 4441.455078125,
+ "status": "excluded 20261006: r01_ira_compensation",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r01_ira_compensation): lands on $4,441.46 vs target $4,441.46. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9801)."
+ },
+ {
+ "scenario_id": "scenario_064",
+ "state": "WI",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 4605.9970703125,
+ "new": 4599.62158203125,
+ "status": "excluded 20261006: r01_ira_compensation",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r01_ira_compensation): lands on $4,599.62 vs target $4,599.62. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9801). It was rechecked on 2026-09-29: read that reason in the first engine_upgrade revision."
+ },
+ {
+ "scenario_id": "scenario_067",
+ "state": "IN",
+ "variable": "local_income_tax",
+ "previous": 0.0,
+ "new": 1348.83203125,
+ "status": "scored",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "new_exclusions",
+ "call": "NEW EXCLUSION (unlisted input: county) - confirm the county the engine assumes in the build trace and the alternative value"
+ },
+ {
+ "scenario_id": "scenario_076",
+ "state": "ID",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 6818.34423828125,
+ "new": 6828.34423828125,
+ "status": "scored",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "approved",
+ "call": "APPROVE (+$10 permanent building fund tax, id_pbf in the 2.36.0 trace) - confirm the trace and the Idaho Code citation"
+ },
+ {
+ "scenario_id": "scenario_080",
+ "state": "PA",
+ "variable": "snap",
+ "previous": 3596.039794921875,
+ "new": 3576.0,
+ "status": "excluded 20261006: r30_snap_heat_and_eat_sua",
+ "hub_audited_target": "no (target 3240.00)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r30_snap_heat_and_eat_sua, off target by $336.00) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_081",
+ "state": "MA",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 8238.40625,
+ "new": 8232.900390625,
+ "status": "excluded 20261006: r22_ma_part_a_loss_offset",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r22_ma_part_a_loss_offset lands on target, but the record's note names an unlisted input too): regenerate only if a reviewer rules that reason out"
+ },
+ {
+ "scenario_id": "scenario_082",
+ "state": "NY",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 9563.052734375,
+ "new": 9579.767578125,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g): lands on $9,579.77 vs target $9,579.77. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032). It was rechecked on 2026-09-29: read that reason in the first engine_upgrade revision."
+ },
+ {
+ "scenario_id": "scenario_082",
+ "state": "NY",
+ "variable": "state_refundable_credits",
+ "previous": 667.0,
+ "new": 1187.5478515625,
+ "status": "excluded in Haiku release (d994/d1022): ny_cdcc_606_c2",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (the release's ruling engine defect ny_cdcc_606_c2): lands on $1,187.55 vs target $1,187.61. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#9948)."
+ },
+ {
+ "scenario_id": "scenario_091",
+ "state": "WI",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 843.6614990234375,
+ "new": 895.8131103515625,
+ "status": "excluded 20261006: r32_wi_capital_gain_distributions",
+ "hub_audited_target": "no (target 878.52)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r32_wi_capital_gain_distributions, off target by $17.30) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_099",
+ "state": "CA",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 10679.75,
+ "new": 10711.484375,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g): lands on $10,711.48 vs target $10,711.48. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032)."
+ },
+ {
+ "scenario_id": "scenario_099",
+ "state": "CA",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 4493.7431640625,
+ "new": 4588.48388671875,
+ "status": "excluded 20261006: r02_ira_219g+r11_ca_itemized_conformity",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g+r11_ca_itemized_conformity): lands on $4,588.48 vs target $4,588.48. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034). It was rechecked on 2026-09-29: read that reason in the first engine_upgrade revision."
+ },
+ {
+ "scenario_id": "scenario_100",
+ "state": "MT",
+ "variable": "snap",
+ "previous": 8625.8896484375,
+ "new": 8556.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "no (target 6924.00)",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_110",
+ "state": "OH",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 23897.443359375,
+ "new": 25747.7734375,
+ "status": "excluded 20261006: r02_ira_219g+r03_estate_income",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g+r03_estate_income): lands on $25,747.77 vs target $25,747.77. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027)."
+ },
+ {
+ "scenario_id": "scenario_110",
+ "state": "OH",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 4057.47216796875,
+ "new": 4236.8076171875,
+ "status": "excluded 20261006: r02_ira_219g+r03_estate_income",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g+r03_estate_income): lands on $4,236.81 vs target $4,236.81. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10027)."
+ },
+ {
+ "scenario_id": "scenario_112",
+ "state": "TX",
+ "variable": "snap",
+ "previous": 287.68316650390625,
+ "new": 0.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_118",
+ "state": "NY",
+ "variable": "snap",
+ "previous": 2903.9404296875,
+ "new": 2868.0,
+ "status": "excluded 20261006: reference_depends_on_unlisted_input",
+ "hub_audited_target": "",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 reference_depends_on_unlisted_input) - check the drafted reason"
+ },
+ {
+ "scenario_id": "scenario_120",
+ "state": "CT",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "previous": 40021.81640625,
+ "new": 40423.17578125,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "excluded_rechecked",
+ "call": "RECHECK (20261006 engine defect r02_ira_219g lands on target, but the record's note names an unlisted input too): regenerate only if a reviewer rules that reason out"
+ },
+ {
+ "scenario_id": "scenario_120",
+ "state": "CT",
+ "variable": "state_income_tax_before_refundable_credits",
+ "previous": 11917.18359375,
+ "new": 11919.0576171875,
+ "status": "excluded 20261006: r02_ira_219g",
+ "hub_audited_target": "yes",
+ "decision": "PROPOSED",
+ "placed_in": "regenerated_exclusions",
+ "call": "REGENERATE (20261006 engine defect r02_ira_219g): lands on $11,919.06 vs target $11,919.06. Confirm the fix is released upstream and name it in upstream (PolicyEngine/policyengine-us#10032)."
+ }
+ ],
+ "note": "Reviewed 2026-10-10 on policyengine-us 2.38.6: every action is the pre-T rehearsal's (main + #10032 + #10034, the code 2.38.6 carries; all 1,984 computed outputs identical), whose calls were reviewed before T: 19 regenerations of engine-defect records the release fixes, each naming its upstream pull request; 16 rechecks of excluded outputs that stay excluded; the two Indiana county exclusions; the Idaho permanent building fund approval."
+}
\ No newline at end of file
diff --git a/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.json b/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.json
new file mode 100644
index 00000000..94120f4f
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.json
@@ -0,0 +1,25 @@
+{
+ "scenario_003|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $22,400.65 for this joint filer household. IRS gross income of $184,872.25 comprises $94,720.00 of taxable retirement distributions, $50,583.25 of taxable Social Security, $32,525.29 of taxable pension income, $4,304.00 of dividends, and $2,739.71 of taxable interest. A $1,582.00 above-the-line loss deduction reduces AGI to $183,290.25, and the $32,200.00 standard deduction reduces taxable income to $151,090.25. The calculation taxes $3,760.00 of qualified dividends at preferential rates, outside the main-rate calculation. Main-rate tax of $21,836.65 plus capital gains tax of $564.00 gives $22,400.65. Modified AGI is $183,290.25, and net investment income tax is $0.00. The listed additional taxes, penalties, and capped non-refundable credits are also zero.",
+ "scenario_005|state_income_tax_before_refundable_credits": "PolicyEngine calculated California income tax before refundable credits of $40,975.25 for this joint filer household. Federal AGI is $543,076.00, and California AGI is $538,928.31 after a $4,147.68 SALT-refund subtraction. California itemized deductions include $17,280.00 of real estate taxes, $2,573.53 of charitable deductions, and $2,986.30 of miscellaneous deductions, totaling $22,839.83 before limitation. After a $2,319.90 overall reduction, itemized deductions are $20,519.93, exceeding the $11,412.00 standard deduction. Subtracting the selected $20,519.93 deduction from California AGI gives taxable income of $518,408.38. California income tax before credits is $41,089.25. Subtracting the $114.00 non-refundable exemption credit gives the reference value of $40,975.25.",
+ "scenario_008|payroll_tax": "PolicyEngine calculated payroll tax of $2,276.66 by summing employee payroll taxes. One of the household's eight people has $26,800.00 of employment income, and the other seven have none. Employee Social Security tax is $1,661.60, equal to 6.2% of $26,800.00, and employee Medicare tax is $388.60, equal to 1.45% of that wage amount. Together they total $2,050.20. New Jersey employee state payroll tax adds $226.46: $50.92 for temporary disability insurance, $61.64 for family leave insurance, $102.51 for unemployment insurance, and $11.39 for the workforce fund. Each contribution’s taxable-wage input is $26,800.00. Adding these four contributions to Social Security and Medicare taxes gives the reference payroll tax of $2,276.66.",
+ "scenario_015|local_income_tax": "PolicyEngine calculated local income tax of $670.55 using Adams County, Indiana, and Indiana AGI of $41,909.38. Employment income of $45,000.00 less $1,041.93 of pre-tax 401(k) contributions gives IRS employment income of $43,958.07. Adding $2,000.00 of taxable interest gives IRS gross income of $45,958.07. The household has a $4,510.00 long-term capital loss, of which the calculation deducts $3,000.00. Together with a $48.69 traditional IRA deduction, above-the-line deductions total $3,048.69, leaving federal AGI of $42,909.38. Indiana exemptions of $1,000.00 reduce this to Indiana AGI of $41,909.38. The county tax is $670.55, and all other listed local income taxes are zero.",
+ "scenario_018|state_income_tax_before_refundable_credits": "PolicyEngine calculated Arizona income tax before refundable credits of $1,137.30 for this single filer in 2026. Arizona adjusted gross income is $61,592.09. Arizona's 2026 standard deduction for a single filer is $16,100: the $15,750 HB 4168 set, indexed as the federal basic standard deduction is (A.R.S. 43-1041(H)). It exceeds the household's $10,637.30 of itemized deductions, so taxable income is $45,492.09, and Arizona's 2.5% rate gives $1,137.30.",
+ "scenario_022|state_income_tax_before_refundable_credits": "PolicyEngine calculated California income tax before refundable credits of $1,968.81 for this household with a 77-year-old head. Federal AGI of $109,149.16 is reduced by $17,624.75 of taxable Social Security to California AGI of $91,524.41. Itemized deductions total $30,389.97: $8,829.00 of real estate taxes, $15,393.41 of charitable deductions, and $6,167.56 of miscellaneous deductions. The charitable amount equals $14,693.41 of cash donations plus $700.00 of non-cash donations. Itemized deductions exceed the $5,706.00 California standard deduction. Subtracting them from California AGI gives taxable income of $61,134.44. Tax before credits is $2,274.81; the $306.00 exemption credit, calculated with an aged/blind count of one, gives the final $1,968.81.",
+ "scenario_025|state_income_tax_before_refundable_credits": "PolicyEngine calculated Ohio income tax before refundable credits of $1,916.61 for this joint return in 2026. Ohio deducts unreimbursed medical care above 7.5% of federal adjusted gross income (R.C. 5747.01(A)(10)). The head paid $6,500 of health insurance premiums and $800 of other medical expenses. Because the household can get an employer-paid plan, the premiums go on line 3 of Ohio's unreimbursed medical care worksheet and the other expenses on line 4, $7,300 in all. Federal adjusted gross income is $94,925.30, so the 7.5% floor is $7,119.40 and the deduction is $180.60, which leaves Ohio adjusted gross income of $94,744.70. Less $3,800 of personal exemptions, taxable nonbusiness income is $90,944.70, taxed at $2,116.61. A $200 pension-based retirement income credit, for the spouse's $32,200 of taxable pension income, lowers the tax to $1,916.61.",
+ "scenario_039|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $8,596.03 for this household in 2026. IRS gross income is $85,469.25, comprising $30,689.25 of taxable Social Security benefits, $26,800.00 of taxable retirement distributions, $2,030.00 of taxable pension income, and $25,950.00 of estate income. A self-employment loss of $6,260.03 is deducted above the line, reducing AGI to $79,209.22. The $16,100.00 standard deduction exceeds the $3,793.91 itemized taxable-income deduction and is selected, leaving taxable income of $63,109.22. Main-rate tax is $8,596.03. Estate income of $25,950.00 also supplies net investment income, but net investment income tax is $0.00. The other listed additions and capped non-refundable credits are zero, leaving $8,596.03.",
+ "scenario_039|state_income_tax_before_refundable_credits": "PolicyEngine calculated Virginia income tax before refundable credits of $1,975.80 for this single filer in 2026. Gross income is $85,469.25: $30,689.25 of taxable Social Security benefits, $26,800 of taxable IRA distributions, $2,030 of taxable pension income and $25,950 of estate and trust income, which federal gross income includes (26 U.S.C. 61(a)(14)). A $6,260.03 self-employment loss leaves federal adjusted gross income of $79,209.22. Virginia subtracts the taxable Social Security benefits, for Virginia adjusted gross income of $48,519.97. Less the $8,750 standard deduction and a $930 personal exemption, taxable income is $38,839.97, taxed at $1,975.80.",
+ "scenario_043|state_refundable_credits": "PolicyEngine calculated Colorado state refundable credits of $0.00 for this household in 2026. The household meets the eligibility rules for Colorado's sales tax refund, but C.R.S. 39-22-2003(2) pays that refund for a tax year only if the state had excess revenues in the fiscal year ending in that tax year. The forecasts published before the reference freeze put fiscal year 2025-26 below the Referendum C cap, with no tax year 2026 refund, so PolicyEngine pays none. The household's other Colorado refundable credits are also $0, so its state refundable credits total $0.00.",
+ "scenario_049|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $30,702.59 for this joint return in 2026. Employment income of $253,000.00 less $15,436.00 of pre-tax contributions gives $237,564.00 of IRS employment income. Adding $3,501.00 of taxable interest and $6,320.00 of dividends gives IRS gross income and AGI of $247,385.00. Taxable-income deductions are $57,779.77, including the $32,200.00 standard deduction, $25,000.00 overtime deduction, and $579.76 charitable deduction for non-itemizers. Taxable income is $189,605.23. Excluding $6,208.00 of qualified dividends leaves $183,397.23 for main rates, producing $29,771.39 of tax. Capital gains tax adds $931.20, giving $30,702.59 before credits. Net investment income tax is zero, with modified AGI of $247,385.00 and net investment income of $9,821.00. Other listed additions and capped non-refundable credits are zero.",
+ "scenario_064|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $4,441.46 for this Wisconsin joint return in 2026. Gross income is $134,329.59: the head's $97,295 of wages less $2,315.40 of pre-tax contributions ($94,979.60), $19,350 of partnership and S corporation income and $20,000 of taxable retirement distributions. Above-the-line deductions total $19,908.19: a $19,800 loss deduction, for the $16,800 farm loss and $3,000 of the $18,235.41 capital loss, and the head's $108.19 traditional IRA contribution. A dependent's $18.03 IRA contribution stays off the filers' return. That leaves adjusted gross income of $114,421.41. Less the $32,200 standard deduction, a $510 qualified business income deduction and a $13,899.29 overtime deduction, taxable income is $67,812.12, taxed at $7,641.46. The child tax credit, $2,200 for the child and $500 for each of two other dependents, takes off $3,200, leaving $4,441.46.",
+ "scenario_064|state_income_tax_before_refundable_credits": "PolicyEngine calculated Wisconsin income tax before refundable credits of $4,599.62 for this joint return in 2026. Wisconsin adjusted gross income equals federal adjusted gross income, $114,421.41: it deducts the $16,800 farm loss, $3,000 of the $18,235.41 capital loss (Wisconsin's limit for a joint return is also $3,000) and the head's $108.19 traditional IRA contribution, but not a dependent's $18.03 contribution, which stays off the filers' return. Less the $8,953.27 sliding-scale standard deduction and $3,500 of exemptions for five people, taxable income is $101,968.14, taxed at $4,599.62 under Wisconsin's 2026 rates, with no nonrefundable credits.",
+ "scenario_067|local_income_tax": "PolicyEngine computed local income tax of $1,348.83 for this Indiana household, using Adams County. Federal gross income and AGI are both $87,302.00, comprising $82,360.00 of employment income and $4,942.00 of taxable pension income. Indiana applies $3,000.00 of exemptions for a tax unit of three people, leaving Indiana AGI of $84,302.00. County tax of $1,348.83 corresponds to 1.6% of that amount: $84,302.00 × 1.6% = $1,348.832, which rounds to $1,348.83. No other local income tax applies, so the Indiana county tax is the full local income tax of $1,348.83.",
+ "scenario_076|state_income_tax_before_refundable_credits": "PolicyEngine calculated Idaho state income tax before refundable credits of $6,828.34 for this head-of-household filer in 2026. Federal adjusted gross income is $164,220. Less the $24,150 standard deduction, $1,000 of Idaho subtractions and an $800 qualified business income deduction, Idaho taxable income is $138,270, and Idaho income tax is $6,818.34, with no nonrefundable credits. State income tax adds Idaho's $10 permanent building fund tax, for $6,828.34.",
+ "scenario_082|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $9,579.77 for this head-of-household filer with one child in 2026. IRS gross income is $120,661.12, comprising $100,013.62 of IRS employment income after $181.37 of pre-tax contributions, $20,480.00 of dividends, $100.00 of taxable interest, and $67.50 of SALT refund income. The household also has a $30,710.82 net capital loss, of which a $3,000.00 loss deduction reduces gross income to AGI of $117,661.12. The $24,150.00 standard deduction plus $700.00 charitable deduction for non-itemizers leaves taxable income of $92,811.12. The calculation excludes $13,424.00 of qualified dividends from main-rate income. Main-rate tax of $10,366.17 plus capital gains tax of $2,013.60 gives $12,379.77 before credits. Subtracting $2,800.00 of non-refundable credits, comprising $600.00 of child and dependent care credit and $2,200.00 of child tax credit, gives $9,579.77.",
+ "scenario_082|state_income_tax_before_refundable_credits": "PolicyEngine calculated New York income tax before refundable credits of $5,599.14 for this head-of-household filer with one child dependent in 2026. IRS gross income of $120,661.12 comprises $100,013.62 of IRS employment income, $20,480.00 of dividends, $100.00 of taxable interest, and $67.50 of SALT refund income. The household has a $30,710.82 capital loss, of which a $3,000.00 above-the-line loss deduction applies, leaving federal AGI of $117,661.12. Subtracting $67.50 of SALT refund income gives New York AGI of $117,593.62. The selected $11,200.00 standard deduction exceeds itemized deductions of $9,328.22. Subtracting the standard deduction and $1,000.00 dependent exemption gives taxable income of $105,393.62. Main New York tax of $5,442.63 plus supplemental tax of $156.51 gives the final $5,599.14.",
+ "scenario_082|state_refundable_credits": "PolicyEngine calculated New York state refundable credits of $1,187.55 for this head-of-household return in 2026: a $307 Empire State child credit and an $880.55 child and dependent care credit. The child credit is its $1,000 base less a $693 phase-out at federal adjusted gross income of $117,661.12. The care credit follows Tax Law 606(c-2), which applies from 2026: $3,000 of the household's $9,600 of child care expenses qualify for its one child, and the applicable percentage at New York adjusted gross income of $117,593.62, about 29.352%, gives $880.55.",
+ "scenario_085|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $2,322.85 for this household in 2026. AGI of $45,573.74 comprises $1,675.00 of taxable interest, $20,720.00 of taxable private pension income, $14,050.74 of taxable Social Security, and $9,128.00 of taxable retirement distributions. Selected taxable-income deductions total $24,150.00: the $18,150.00 standard deduction plus a $6,000.00 additional senior deduction. This exceeds the $10,115.88 deduction total available if itemizing. Subtracting $24,150.00 from AGI leaves taxable income of $21,423.74, producing $2,322.85 of main-rate tax and tax before credits. Net investment income tax is zero, with modified AGI of $45,573.74 and net investment income of $1,675.00 as inputs. Other listed additional taxes and capped non-refundable credits are zero, leaving the final tax of $2,322.85.",
+ "scenario_099|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $10,711.48 for this joint filer household in 2026. IRS gross income is $163,066.80, comprising $161,912.80 of IRS employment income after $3,087.20 of pre-tax contributions, $1,010.00 of taxable interest, and $144.00 of dividends. A $6,478.00 net capital loss is separately recorded. Above-the-line deductions of $3,337.50, comprising a $3,000.00 loss deduction and $337.50 educator expense deduction, give AGI of $159,729.30. The $32,200.00 standard deduction and $2,000.00 charitable deduction for non-itemizers leave taxable income of $125,529.30. After excluding $128.00 of qualified dividends, main-rate tax is $17,012.29. Capital gains tax is $19.20, and total tax before credits is $17,031.48. Non-refundable credits total $6,320.00: a $1,920.00 child and dependent care credit, calculated as 32% of $6,000.00 in expenses capped from $12,740.00, plus a $4,400.00 child tax credit. The final amount is $10,711.48.",
+ "scenario_110|federal_income_tax_before_refundable_credits": "PolicyEngine calculated federal income tax before refundable credits of $25,747.77 for this household in 2026, combining $19,615.21 of main-rate tax and $6,132.56 of capital gains tax. IRS gross income and AGI are both $169,942.94. Income inputs include $95,755.10 of IRS employment income after pre-tax contributions, $30,869.77 of capital gains, $21,301.00 of taxable interest, $15,694.12 of dividends, and $6,322.94 of estate income. Selected deductions of $16,487.47, comprising the $16,100.00 standard deduction and $387.47 charitable deduction, leave taxable income of $153,455.47. The engine excludes $40,883.77 from main-rate income, leaving $112,571.70. The excluded amount comprises $30,851.77 of long-term capital gains and $10,032.00 of qualified dividends and feeds the separate capital gains calculation. Net investment income tax, other listed additional taxes, and capped non-refundable credits are zero.",
+ "scenario_110|state_income_tax_before_refundable_credits": "PolicyEngine calculated Ohio income tax before refundable credits of $4,236.81 for this household in 2026. IRS gross income and person-level AGI are both $169,942.94. This amount flows through the person-level Ohio AGI calculation into Ohio AGI of $169,942.94. Subtracting $1,900.00 of Ohio personal exemptions gives taxable income of $168,042.94, also reported as taxable nonbusiness income. Tax on this income is $4,236.81 before non-refundable credits, and Ohio income tax before refundable credits remains $4,236.81. Every other state’s component shown is zero, so aggregate state income tax before refundable credits is also $4,236.81.",
+ "scenario_120|state_income_tax_before_refundable_credits": "PolicyEngine calculated Connecticut income tax before refundable credits of $11,919.06 for this household in 2026. IRS gross income is $236,860.80. Positive income inputs include $164,979.56 of IRS employment income, $35,615.00 of taxable Social Security, $30,216.00 of taxable pension income, $2,300.00 of taxable interest, $2,000.00 of rental income, and $1,750.24 of SALT refund income. A $2,446.59 above-the-line loss deduction leaves federal AGI of $234,414.20. Connecticut AGI subtractions total $26,890.23, comprising a $25,140.00 Social Security benefit adjustment and the SALT refund subtraction. Connecticut AGI and taxable income are $207,523.97. The tax calculation also uses a $250.00 phase-out add-back and $430.00 of recapture adjustments, comprising $250.00 of low-tax recapture and $180.00 of middle-tax recapture. The resulting tax before refundable credits is $11,919.06."
+}
diff --git a/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.md b/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.md
new file mode 100644
index 00000000..39c0f719
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/hand_corrected_narratives.md
@@ -0,0 +1,116 @@
+# Hand-corrected reference narratives
+
+`hand_corrected_narratives.json` replaces the narrative writer (claude-haiku-4-5)
+for every output the engine upgrade changes (23 on policyengine-us 2.38.6),
+through `scripts/narratives_upgrade.py --hand-corrected`. Judges' prompts render
+these narratives. Each text states the value and figures of the final engine's
+trace; the notes below give why each departs from the writer's draft.
+
+The first corrections were made on the policyengine-us 2.37.2 rehearsal, where the
+upgrade changed eight outputs. The independent review of the upgrade machinery
+(GPT-6.1 Sol, 2026-10-09) found two of the written ones wrong (Ohio 025, Colorado
+043). Checking the other changed outputs' narratives against their engine traces
+found six more that misstate a figure or the mechanism behind the change:
+
+- **scenario_064 federal and state.** They counted the household's $126.22 of
+ traditional IRA contributions, but the engine deducts only the head's $108.19.
+ A dependent's $18.03 stays off the filers' return: that is policyengine-us #9801,
+ the fix that regenerates both outputs. The federal one also gave the $3,200
+ child tax credit to one child. It is $2,200 for the child and $500 for each of
+ two other dependents.
+- **scenario_039 Virginia.** It left out the $25,950 of estate and trust income,
+ which federal gross income now includes (#9633). That inclusion regenerates the
+ output.
+- **scenario_018 Arizona.** It called the $16,100 deduction a mix of the standard
+ and itemized deductions, and its AGI arithmetic was off by $50. The $16,100 is
+ the standard deduction, which exceeds the itemized $10,637.30.
+- **scenario_076 Idaho.** It gave federal AGI as $163,220, not $164,220, so its
+ arithmetic did not reach the taxable income it stated.
+- **scenario_082 New York.** (On 2.37.2; its final text is in the section below.) It
+ applied a 29% rate to $3,000, which gives $870, not the $880.61 it stated. The engine's applicable percentage is about 29.354%, read at
+ New York AGI ($117,585.15). The child credit's phase-out reads federal AGI
+ ($117,652.65; `ny_ctc_post_2024_phase_out` uses `adjusted_gross_income`).
+
+Every figure in those corrections was the engine's on policyengine-us 2.37.2, from
+the build's trace or computed with the pinned conventions.
+
+- **scenario_025 Ohio state income tax.** The written narrative said the engine
+ "correctly applied Ohio's rule limiting health insurance premium deductions to
+ Medicare-eligible filers without employer coverage". That restriction is the
+ defect the exclusion recorded and policyengine-us #10020 removed. The
+ correction follows the engine: `oh_insured_unreimbursed_medical_care_expense_amount`
+ (line 3 premiums when the filer can get Medicare or an employer-paid plan, plus
+ line 4 other medical expenses) and `oh_insured_unreimbursed_medical_care_expenses`
+ (the excess over `gov.states.oh.tax.income.deductions.unreimbursed_medical_care_expenses.rate`,
+ 0.075, of federal AGI). Computed on policyengine-us 2.37.2 with the pinned
+ conventions: premiums $6,500, other medical $800, federal AGI $94,925.30,
+ deduction $180.60, Ohio AGI $94,744.70, tax $2,116.61 before a $200 credit.
+- **scenario_043 Colorado state refundable credits.** The written narrative put
+ the $0 down to the household's income. The household meets the refund's
+ eligibility rules (`co_sales_tax_refund_eligible` is true), and the refund is $0
+ because `gov.states.co.tax.income.credits.sales_tax_refund.in_effect` is false
+ for 2026 (C.R.S. 39-22-2003(2): no excess revenues for fiscal year 2025-26), as
+ the exclusion record's corrected value said.
+
+Each text states the reference value to the cent, as the narratives script
+requires, and names no engine version, so a rebuild on a newer engine can reuse
+it while the output's value and trace are unchanged.
+
+## Corrections on the final engine
+
+On the code the final engine carries (checked before its release, then on
+policyengine-us 2.38.6 itself, with identical values), 24 outputs change, and Claude
+Haiku 4.5 wrote a narrative for each from its trace. An independent reviewer (GPT-6.1
+Sol, read-only) then checked every narrative against its trace: each number, each sum,
+and each statement of how the value was derived. It found errors in all 17 that had no
+hand correction; its report is kept with the release's review records, outside the
+repository. The texts in `hand_corrected_narratives.json` for those outputs are its
+trace-only rewrites, reworded for readers. A script then checked that every dollar figure
+in each is a value in the output's trace or a sum or difference of two of them. One of
+the 24, scenario_099 California income tax, then stayed excluded (below), so the release
+installs 23. What each drafted narrative got wrong:
+
+- **scenario_003 federal income tax.** It put the $1,582 capital loss inside gross
+ income; the loss is an above-the-line deduction that takes $184,872.25 to adjusted
+ gross income of $183,290.25.
+- **scenario_005 California income tax.** It subtracted the deductions from federal
+ adjusted gross income, which does not give the taxable income it stated; they come
+ off California AGI ($538,928.31). It also attributed the limitation to conformity
+ rules the trace does not show.
+- **scenario_008 payroll tax.** It said the fix leaves New Jersey's worker unemployment
+ and workforce contributions out of payroll tax. The trace includes both ($102.51 and
+ $11.39). It also named a version label and miscounted the household.
+- **scenario_015 and scenario_067 local income tax.** scenario_015's subtracted the full
+ $4,510 capital loss in gross income, where the calculation deducts $3,000. Both said
+ PolicyEngine assigns the county "per its default convention", which the trace cannot
+ show; it shows only the county used.
+- **scenario_022 California income tax.** It said the charitable deduction applies
+ federal limitations. The deduction equals the donations.
+- **scenario_039 federal income tax.** Only its description of the filer went beyond the
+ trace.
+- **scenario_049, scenario_099 and scenario_110 federal income tax.** Each applied the
+ main rates to all taxable income; the calculation first takes out the qualified
+ dividends and capital gains taxed at preferential rates. scenario_099's also
+ multiplied the 32% care credit rate by $12,740 of expenses, where the credit uses the
+ $6,000 limit, and subtracted the full capital loss.
+- **scenario_082 federal income tax and New York income tax.** Both subtracted the full
+ $30,710.82 capital loss in gross income, where a $3,000 loss deduction applies. The
+ federal one omitted the $700 charitable deduction its taxable income needs, and the
+ New York one stated a household income figure that matches no value in the trace.
+- **scenario_082 New York refundable credits.** The IRA fix changes this household's
+ adjusted gross income, so the credit is $1,187.55 on the final engine, where it was
+ $1,187.61 on policyengine-us 2.37.2. The text states the child credit's phase-out at
+ federal AGI ($117,661.12) and the care credit's percentage at New York AGI
+ ($117,593.62), about 29.352%.
+- **scenario_085 federal income tax.** It gave a threshold as the reason for a $0 net
+ investment income tax, which the trace does not show.
+- **scenario_099 California income tax.** (Not installed: the output stays excluded.) Its
+ draft attributed the $1,256 exemption credit to two children; the trace shows two
+ dependents and no split. The release's judge then found that the reference rests on a
+ further defect: California does not conform to the federal educator expense deduction
+ (FTB Instructions for Schedule CA (540), Section C, line 11), which policyengine-us
+ 2.38.6 still allows, so the output keeps its record (`excluded_outputs_rechecked`).
+- **scenario_110 Ohio income tax.** It said adjusted gross income reflects traditional
+ IRA contributions; none appears, and gross income equals AGI.
+- **scenario_120 Connecticut income tax.** Only its description of the filer went
+ beyond the trace.
diff --git a/reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py b/reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py
new file mode 100644
index 00000000..4ad04ce1
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py
@@ -0,0 +1,1656 @@
+"""Rebuild PolicyBench's US references on a newer policyengine-us (the second
+engine upgrade, after release dashboard-data-20261006).
+
+Generalized from reference_audit/2026-09-28/scripts/build_references_latest.py,
+which moved the references from policyengine-us 1.755.4 to 2.15.17 (PR #182).
+
+Rulings:
+- Max, 2026-09-28: "we should be using the latest pe for this always!"
+- Max, 2026-10-09, on the release that adds Claude Haiku 5.5: "yes i want to wait
+ for hte fixed engine". The references move from 2.15.17 to the newer release
+ that fixes engine defects behind PolicyBench's exclusions; a cell the new
+ engine fixes is regenerated and its exclusion record removed (the #178
+ precedent, cb312fd7).
+- d1022 and d994 (docs/haiku55/spec.json): the release's ten ruled exclusions.
+ The household-scope cells (scenario_123, scenario_093, federal and state) and
+ the Louisiana cells (scenario_051, scenario_077) stay excluded; the four
+ engine-defect cells (AZ 018, OH 025, CO 043, NY 082) are regenerated once the
+ engine lands on their audited corrected values.
+- Rule 5: an excluded output keeps the value it was decided on; an excluded
+ output that moves gets an excluded_rechecked entry.
+
+Base (read from git, never from the checkout, whose snapshot holds whatever was
+last installed): release 20261006 at BASE_COMMIT, i.e. 2.15.17 references and the
+64-record exclusion record, each file pinned by sha256. The release's ten ruled
+records come from scripts/finish_haiku55.py's build_release_exclusions, so the
+74-record release record this script edits is byte-for-byte the one the Haiku
+driver installs. The original v1.1 frozen values (each change's "frozen") come
+from FROZEN_COMMIT.
+
+Method: every output is computed on the installed policyengine-us with
+reference_audit/2026-09-28/fixes/latest_final.py (the pre-freeze conventions and
+the Maryland local-scope adapter), imported in place from --fixes-dir, with the
+households built by reference_audit/2026-09-28/scripts/sweep.py's
+build_situation. Every fix module's sha256 must equal the one the 2026-09-29
+engine_upgrade revision records; latest_final.py, sweep.py and the sales tax
+tables (r19_irs_sales_tax_2025.json, which latest_c_irs_sales_tax_2025.py reads
+from its own directory) must each equal the bytes committed at BASE_COMMIT (the
+tables as reference_audit/2026-09-22/fixes holds them).
+
+A move is a change beyond the exact-match tolerance
+(policybench.paper_results.moves_beyond_tolerance: more than $1 for an amount,
+any change for a 0/1 flag). A scored output takes the computed value; one that
+changes by $1 or less (more than EPS) is recorded as engine_upgrade_within_1.
+
+ACTIONS FILE (JSON; every list optional except where a move needs it):
+
+ engine "policyengine-us X.Y.Z"; must equal the installed release.
+ previous_engine "policyengine-us 2.15.17"; must equal the base sidecar's
+ last engine_upgrade engine_version.
+ date UTC day of the build (YYYY-MM-DD): the revision's date, the
+ derivation's, and each new record's decided_on. The sidecar's
+ regenerated_at_utc must fall on it.
+ release_spec_sha256 sha256 of docs/haiku55/spec.json the actions were drafted
+ against.
+ draft true for actions_from_cells.py's output; the builder
+ refuses a draft unless --allow-draft (rehearsals only).
+ review, note free text for reviewers; ignored.
+ approved [{scenario_id, variable, value, cause, basis}]: scored
+ outputs allowed to move; the engine must give value within
+ APPROVED_TOL (half a cent).
+ regenerated_exclusions [{scenario_id, variable, alternative_value,
+ tolerance, upstream, basis, target}]: excluded outputs (in
+ 20261006's record or the release's ruled records) whose
+ reference_engine_defect the new engine fixes. alternative_value
+ must be the record's; tolerance is at most $1; upstream must
+ name the fix ("to be filed" does not). The engine must land
+ within the tolerance of the audited corrected value (a flag
+ must equal it), which target says how to know:
+ {"kind": "record"} (the default): the record's
+ alternative_value. Right when nothing else in the engine
+ has moved the output since the record was decided.
+ {"kind": "fix_modules", "modules": [file, ...],
+ "evidence": {"path", "sha256"}}: the record's audited fix
+ modules (reference_audit/2026-09-22/fixes, the bytes
+ committed at BASE_COMMIT), on a recent engine that still
+ has the defect. The evidence file (--evidence-request
+ mode, on that engine) gives the output there without and
+ with the modules; they must move it beyond the tolerance
+ (the defect was there), and the corrected value is the
+ target. On the new engine the builder applies the modules
+ again: they must move it by no more than the tolerance
+ (nothing is left to fix). A record decided on an older
+ engine needs this kind once other engine changes have
+ moved the output, because its alternative_value no longer
+ includes them.
+ The record is removed and the output scored at the engine
+ value.
+ new_exclusions [complete exclusion record]: scored outputs the new engine
+ moves onto an input the prompt does not state (for example
+ Indiana county). frozen_value must be the engine value
+ (within 1e-3), engine_version the new engine, decided_on the
+ date, and the alternative must differ beyond the tolerance.
+ excluded_rechecked [{scenario_id, variable, reason}]: excluded outputs that
+ move but stay excluded, keeping the decided value.
+ kept_exclusions_from_release [{scenario_id, variable}]: the release's ruled
+ outputs that stay excluded. Together with the ruled outputs
+ listed in regenerated_exclusions they must be exactly the
+ spec's ruled outputs. A kept output that moves is also
+ listed in excluded_rechecked, the one pair of lists that
+ may share an output.
+
+The builder refuses, listing every problem with exact engine values: an
+unexplained move (scored or excluded), an approved output that does not move or
+lands elsewhere, a regenerated output off its target or not an engine defect, a
+new exclusion on an output that does not move, a rechecked output that does not
+move, an entry on an unknown output, and an output named by two actions.
+
+It writes to --out-dir:
+ reference_outputs.csv 20261006's rows byte for byte, except each changed row,
+ whose value field is rewritten.
+ reference_exclusions.json the release record (20261006's 64 + the ruled ten)
+ less the regenerated records, with the new exclusions
+ before the trailing 2026-09-29 audit record and one
+ sentence appended to the derivation naming the upgrade.
+ reference_outputs.csv.meta.json the base sidecar with the US bundle, csv
+ sha256 and regenerated_at_utc updated and a second
+ engine_upgrade revision appended; every earlier revision
+ is kept byte for byte.
+ reference_traces.json engine traces of every changed output.
+ computed.csv (also with --computed-csv, written before any refusal)
+ every output on the new engine, in sweep.py's columns.
+
+Run (PYTHONPATH is the checkout; the fixes directory needs the sales tax tables):
+
+ PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\
+ /bin/python \\
+ reference_audit/2026-10-09-engine-upgrade/scripts/build_references_upgrade.py \\
+ --actions --out-dir [--fixes-dir ] [--allow-draft]
+
+EVIDENCE MODE (on a pre-fix engine, for "fix_modules" targets):
+
+ ... build_references_upgrade.py --evidence-request \\
+ --evidence-out [--fixes-dir ]
+
+The request is {"items": [{scenario_id, variable, modules: [file, ...]}]}. The
+evidence file names the installed engine and every module's sha256, and gives
+each item's engine_value (the pinned conventions alone) and corrected_value
+(the conventions plus the modules, applied after them).
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import csv
+import datetime
+import hashlib
+import importlib.util
+import io
+import json
+import math
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[2]
+RUN = "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace"
+# The merge of PR #202 on main, whose tree holds release 20261006.
+BASE_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f"
+BASE_SHA256 = {
+ "reference_outputs.csv": (
+ "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466"
+ ),
+ "reference_outputs.csv.meta.json": (
+ "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b"
+ ),
+ "reference_exclusions.json": (
+ "92741dfd047298eacb79a0dadeb7e4965e3c6f986beab0341b12ff8f5b023815"
+ ),
+ "scenarios.csv": (
+ "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a"
+ ),
+}
+# Re-freeze on the 24-model board (#121): the snapshot's last commit holding the
+# v1.1 frozen references (policyengine-us 1.755.4), the bundle the 2026-09-29
+# builder read as "frozen".
+FROZEN_COMMIT = "f108a959d91567c050a99d4d1d583904b3d5a809"
+FROZEN_SHA256 = "b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368"
+PREVIOUS_ENGINE = "policyengine-us 2.15.17"
+FIXES_REL = "reference_audit/2026-09-28/fixes"
+FIXES = ROOT / FIXES_REL
+HARNESS_REL = "reference_audit/2026-09-28/scripts/sweep.py"
+HARNESS = ROOT / HARNESS_REL
+FIX_ENTRY = "latest_final"
+SALES_TAX_TABLES = "r19_irs_sales_tax_2025.json"
+SALES_TAX_REL = f"reference_audit/2026-09-22/fixes/{SALES_TAX_TABLES}"
+SALES_TAX_SOURCE = ROOT / SALES_TAX_REL
+# The audited root-cause fix modules (2026-09-22): a "fix_modules" target
+# applies them after the conventions.
+AUDIT_FIXES_REL = "reference_audit/2026-09-22/fixes"
+AUDIT_FIXES = ROOT / AUDIT_FIXES_REL
+EVIDENCE_KIND = "regeneration_evidence"
+# A fix module loads a sibling as Path(__file__).with_name(""); those
+# siblings run too, so they are pinned with it (module_dependencies, from
+# policybench.fix_module_closure, which refuses any other way of loading one).
+TARGET_RECORD = "record"
+TARGET_FIX_MODULES = "fix_modules"
+YEAR = 2026
+EPS = 1e-6
+FROZEN_TOL = 1e-3
+APPROVED_TOL = 0.005
+MAX_REGENERATION_TOL = 1.0
+# The trailing record of 20261006's exclusion record: the 2026-09-29 audit's
+# exclusion, which tests/test_reference_upgrade.py requires last.
+AUDIT_TAIL = ("scenario_023", "head_medicaid_eligible")
+ENGINE_DEFECT = "reference_engine_defect"
+# A regeneration claims a released upstream fix and must name it.
+UNNAMED_UPSTREAM = frozenset({"", "to be filed", "none", "unknown"})
+ACTION_KEYS = frozenset(
+ {
+ "engine",
+ "previous_engine",
+ "date",
+ "release_spec_sha256",
+ "draft",
+ "review",
+ "note",
+ "approved",
+ "regenerated_exclusions",
+ "new_exclusions",
+ "excluded_rechecked",
+ "kept_exclusions_from_release",
+ }
+)
+RULE = (
+ "A scored reference follows from the stated facts and from law published before "
+ "the 2026-07-03 reference freeze."
+)
+ENGINE_RULE = (
+ "References come from the newest policyengine-us release when PolicyBench begins "
+ "the reference sweep; at publication PolicyBench checks that the newest release "
+ "gives the same values."
+)
+UPGRADE_RULE = (
+ "Max, 2026-10-09, on waiting for the release that fixes the engine defects "
+ "behind PolicyBench's exclusions: 'yes i want to wait for hte fixed engine'. An "
+ "excluded output whose engine defect the new release fixes, landing on its "
+ "audited corrected value, is regenerated and its exclusion record removed; every "
+ "other excluded output keeps the value it was decided on."
+)
+NUMBER_WORDS = {
+ 1: "one",
+ 2: "two",
+ 3: "three",
+ 4: "four",
+ 5: "five",
+ 6: "six",
+ 7: "seven",
+ 8: "eight",
+ 9: "nine",
+ 10: "ten",
+}
+
+Key = tuple[str, str]
+
+
+class Refusal(SystemExit):
+ """The builder refuses to write references."""
+
+
+def sha256_bytes(raw: bytes) -> str:
+ return hashlib.sha256(raw).hexdigest()
+
+
+def sha256(path: Path) -> str:
+ return sha256_bytes(Path(path).read_bytes())
+
+
+def dump_record(value: dict) -> str:
+ """The serialization of the sidecar and the exclusion record."""
+ return json.dumps(value, indent=2) + "\n"
+
+
+def words(n: int) -> str:
+ return NUMBER_WORDS.get(n, str(n))
+
+
+def key_of(item: dict) -> Key:
+ return (str(item["scenario_id"]), str(item["variable"]))
+
+
+def engine_number(engine: str) -> str:
+ prefix = "policyengine-us "
+ if not engine.startswith(prefix):
+ raise Refusal(f"engine must read 'policyengine-us X.Y.Z', not {engine!r}")
+ return engine.removeprefix(prefix)
+
+
+def engine_older(older: str, newer: str) -> bool:
+ """Whether one 'policyengine-us X.Y.Z' release precedes another."""
+
+ def parts(engine: str) -> tuple[int, ...]:
+ return tuple(int(p) for p in engine_number(engine).split("."))
+
+ return parts(older) < parts(newer)
+
+
+def beyond(variable: str, before: float, after: float) -> bool:
+ """A move beyond the exact-match tolerance, as the paper counts it."""
+ from policybench.paper_results import moves_beyond_tolerance
+
+ return moves_beyond_tolerance(variable, before, after)
+
+
+# --- Base ------------------------------------------------------------------
+
+
+def git_blob(commit: str, path: str, root: Path = ROOT) -> bytes:
+ result = subprocess.run(
+ ["git", "-C", str(root), "show", f"{commit}:{path}"], capture_output=True
+ )
+ if result.returncode:
+ raise Refusal(
+ f"cannot read {path} at {commit[:12]} (fetch full history): "
+ f"{result.stderr.decode().strip()}"
+ )
+ return result.stdout
+
+
+@dataclass
+class Row:
+ key: Key
+ fields: list[str]
+ raw: str
+
+ @property
+ def value(self) -> float:
+ return float(self.fields[2])
+
+
+@dataclass
+class ReferenceCsv:
+ header_raw: str
+ header: list[str]
+ rows: list[Row]
+
+ def values(self) -> dict[Key, float]:
+ return {row.key: row.value for row in self.rows}
+
+
+def physical_lines(text: str) -> list[str]:
+ """Split on '\\n' only, each piece keeping its terminator."""
+ lines = text.split("\n")
+ out = [line + "\n" for line in lines[:-1]]
+ if lines[-1]:
+ out.append(lines[-1])
+ return out
+
+
+def parse_reference_csv(text: str) -> ReferenceCsv:
+ """Rows with their raw text, so an unchanged row is written byte for byte."""
+ lines = physical_lines(text)
+ reader = csv.reader(iter(lines))
+ records, used = [], 0
+ for fields in reader:
+ records.append((fields, "".join(lines[used : reader.line_num])))
+ used = reader.line_num
+ (header, header_raw), body = records[0], records[1:]
+ if header[:3] != ["scenario_id", "variable", "value"]:
+ raise Refusal(f"unexpected reference header {header}")
+ rows = [Row((f[0], f[1]), f, raw) for f, raw in body]
+ if len({row.key for row in rows}) != len(rows):
+ raise Refusal("the reference CSV lists an output twice")
+ return ReferenceCsv(header_raw, header, rows)
+
+
+def format_row(fields: list[str]) -> str:
+ buffer = io.StringIO()
+ csv.writer(buffer, lineterminator="\n").writerow(fields)
+ return buffer.getvalue()
+
+
+def render_reference_csv(reference: ReferenceCsv, new_values: dict[Key, float]) -> str:
+ """The base CSV with each output in new_values rewritten at that value."""
+ parts = [reference.header_raw]
+ for row in reference.rows:
+ if row.key in new_values:
+ fields = list(row.fields)
+ fields[2] = repr(float(new_values[row.key]))
+ raw = format_row(fields)
+ if not row.raw.endswith("\n"):
+ raw = raw.removesuffix("\n")
+ parts.append(raw)
+ else:
+ parts.append(row.raw)
+ return "".join(parts)
+
+
+@dataclass
+class Base:
+ reference: ReferenceCsv
+ meta: dict
+ exclusions: dict
+ scenarios_csv: str
+ frozen: dict[Key, float]
+
+
+def load_base(root: Path = ROOT) -> Base:
+ """Release 20261006's references at BASE_COMMIT, each file pinned."""
+ blobs = {}
+ for name, pin in BASE_SHA256.items():
+ raw = git_blob(BASE_COMMIT, f"{RUN}/{name}", root)
+ if sha256_bytes(raw) != pin:
+ raise Refusal(f"{name} at {BASE_COMMIT[:12]} does not match its pin")
+ blobs[name] = raw.decode()
+ frozen_raw = git_blob(FROZEN_COMMIT, f"{RUN}/reference_outputs.csv", root)
+ if sha256_bytes(frozen_raw) != FROZEN_SHA256:
+ raise Refusal("the v1.1 frozen references do not match their pin")
+ return Base(
+ reference=parse_reference_csv(blobs["reference_outputs.csv"]),
+ meta=json.loads(blobs["reference_outputs.csv.meta.json"]),
+ exclusions=json.loads(blobs["reference_exclusions.json"]),
+ scenarios_csv=blobs["scenarios.csv"],
+ frozen=parse_reference_csv(frozen_raw.decode()).values(),
+ )
+
+
+def release_record(base_exclusions: dict) -> tuple[dict, set[Key], str]:
+ """The release's 74-record exclusion record, as scripts/finish_haiku55.py
+ builds it from the spec; the ruled outputs; and the spec's sha256."""
+ path = ROOT / "scripts" / "finish_haiku55.py"
+ spec = importlib.util.spec_from_file_location("finish_haiku55_for_upgrade", path)
+ driver = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = driver
+ spec.loader.exec_module(driver)
+ release_spec = driver.load_spec()
+ doc = driver.build_release_exclusions(copy.deepcopy(base_exclusions), release_spec)
+ ruled = {driver.spec_key(r) for r in driver.spec_records(release_spec)}
+ return doc, ruled, sha256(ROOT / driver.SPEC_PATH)
+
+
+# --- The plan ----------------------------------------------------------------
+
+
+@dataclass
+class Plan:
+ problems: list[str] = field(default_factory=list)
+ new_values: dict[Key, float] = field(default_factory=dict)
+ changed: list[dict] = field(default_factory=list)
+ within: list[dict] = field(default_factory=list)
+ rechecked: list[dict] = field(default_factory=list)
+ regenerated: list[dict] = field(default_factory=list)
+ new_records: list[dict] = field(default_factory=list)
+ kept: list[Key] = field(default_factory=list)
+
+
+def _entries(actions: dict, name: str) -> dict[Key, dict]:
+ entries: dict[Key, dict] = {}
+ for item in actions.get(name, []):
+ k = key_of(item)
+ if k in entries:
+ raise Refusal(f"{name} lists {k} twice")
+ entries[k] = item
+ return entries
+
+
+def validate_actions(actions: dict, *, allow_draft: bool = False) -> None:
+ unknown = set(actions) - ACTION_KEYS
+ if unknown:
+ raise Refusal(f"unknown actions keys {sorted(unknown)}")
+ for name in ("engine", "previous_engine", "date"):
+ if not isinstance(actions.get(name), str) or not actions[name]:
+ raise Refusal(f"actions must name {name}")
+ datetime.date.fromisoformat(actions["date"])
+ engine_number(actions["engine"])
+ if actions.get("draft") and not allow_draft:
+ raise Refusal(
+ "the actions file is a draft (actions_from_cells.py output): review "
+ "every judgment call and set draft to false, or pass --allow-draft "
+ "for a rehearsal"
+ )
+ lists = (
+ "approved",
+ "regenerated_exclusions",
+ "new_exclusions",
+ "excluded_rechecked",
+ "kept_exclusions_from_release",
+ )
+ # A kept ruled output that moves is also rechecked (rule 5): kept declares
+ # it stays excluded, rechecked records its move. No other pair may share an
+ # output.
+ allowed = {frozenset({"excluded_rechecked", "kept_exclusions_from_release"})}
+ seen: dict[Key, str] = {}
+ for name in lists:
+ for k in _entries(actions, name):
+ if k in seen and frozenset({seen[k], name}) not in allowed:
+ raise Refusal(f"{k} is in both {seen[k]} and {name}")
+ seen.setdefault(k, name)
+
+
+def target_modules(entry: dict) -> tuple[str, ...]:
+ """The fix modules a regeneration's target names (none for a record)."""
+ target = entry.get("target") or {}
+ modules = target.get("modules") if isinstance(target, dict) else None
+ if isinstance(modules, list) and all(isinstance(m, str) for m in modules):
+ return tuple(modules)
+ return ()
+
+
+def regeneration_target(
+ k: Key,
+ entry: dict,
+ record: dict,
+ value: float,
+ tolerance: float,
+ *,
+ engine: str,
+ evidence: dict[str, dict],
+ module_pins: dict[str, str],
+ module_values: dict[tuple[Key, tuple[str, ...]], float],
+ module_deps: dict[str, list[str]] | None = None,
+) -> tuple[dict | None, list[str]]:
+ """The audited corrected value a regenerated output must land on, and how
+ it is known (the entry's target), or the problems that stop it. Pure."""
+ target = entry.get("target", {"kind": TARGET_RECORD})
+ if not isinstance(target, dict):
+ return None, [f"regenerated {k}: target is not an object"]
+ kind = target.get("kind")
+ if kind == TARGET_RECORD:
+ if set(target) != {"kind"}:
+ return None, [f"regenerated {k}: a record target takes no other fields"]
+ return {
+ "kind": TARGET_RECORD,
+ "value": float(record["alternative_value"]),
+ "engine": record["engine_version"],
+ }, []
+ if kind != TARGET_FIX_MODULES:
+ return None, [f"regenerated {k}: unknown target kind {kind!r}"]
+ modules = target_modules(entry)
+ problems = []
+ if set(target) != {"kind", "modules", "evidence"}:
+ problems.append(f"regenerated {k}: a fix_modules target takes kind, modules "
+ "and evidence")
+ if not modules or len(set(modules)) != len(modules):
+ problems.append(f"regenerated {k}: the target names no modules, or one twice")
+ unpinned = [m for m in modules if m not in module_pins]
+ if unpinned:
+ problems.append(f"regenerated {k}: unknown fix modules {unpinned}")
+ ref = target.get("evidence")
+ doc = evidence.get(ref.get("path")) if isinstance(ref, dict) else None
+ if doc is None:
+ problems.append(f"regenerated {k}: the target's evidence is not loaded")
+ if problems:
+ return None, problems
+ variable = k[1]
+ if not engine_older(doc["engine"], engine):
+ problems.append(
+ f"regenerated {k}: the evidence is computed on {doc['engine']}, not an "
+ f"engine older than {engine}; it must come from one that still has the "
+ "defect"
+ )
+ stale = [m for m in modules if doc["modules"].get(m) != module_pins[m]]
+ if stale:
+ problems.append(f"regenerated {k}: the evidence ran other bytes of {stale}")
+ items = [
+ item
+ for item in doc["items"]
+ if key_of(item) == k and tuple(item["modules"]) == modules
+ ]
+ if len(items) != 1:
+ problems.append(
+ f"regenerated {k}: the evidence has {len(items)} items for {list(modules)}"
+ )
+ return None, problems
+ item = items[0]
+ before, corrected = float(item["engine_value"]), float(item["corrected_value"])
+ dependencies = []
+ for m in modules:
+ for d in (module_deps or {}).get(m, []):
+ if d not in modules and d not in dependencies:
+ dependencies.append(d)
+ unpinned = [
+ d for d in dependencies if d not in module_pins or doc["modules"].get(d) != module_pins[d]
+ ]
+ if unpinned:
+ problems.append(
+ f"regenerated {k}: the evidence does not pin the siblings its modules "
+ f"load, as committed: {unpinned}"
+ )
+ if not (math.isfinite(before) and math.isfinite(corrected)):
+ problems.append(f"regenerated {k}: the evidence's values are not finite")
+ return None, problems
+ if not beyond(variable, before, corrected):
+ problems.append(
+ f"regenerated {k}: on {doc['engine']} the modules move it only "
+ f"{before!r} -> {corrected!r}, so the evidence shows no defect"
+ )
+ after = module_values.get((k, modules))
+ if after is None:
+ problems.append(f"regenerated {k}: the modules were not applied on {engine}")
+ elif not math.isfinite(after) or abs(after - value) > tolerance or beyond(
+ variable, value, after
+ ):
+ problems.append(
+ f"regenerated {k}: the audited fix still moves it on {engine}: "
+ f"{value!r} -> {after!r}"
+ )
+ if problems:
+ return None, problems
+ return {
+ "kind": TARGET_FIX_MODULES,
+ "value": corrected,
+ "engine": doc["engine"],
+ "engine_value": before,
+ "modules": [{"module": m, "sha256": module_pins[m]} for m in modules],
+ "dependencies": [{"module": d, "sha256": module_pins[d]} for d in dependencies],
+ "evidence": ref["path"],
+ "evidence_sha256": ref["sha256"],
+ "value_with_modules": after,
+ }, []
+
+
+def plan_upgrade(
+ base: Base,
+ release: dict,
+ ruled: set[Key],
+ actions: dict,
+ computed: dict[Key, float],
+ *,
+ evidence: dict[str, dict] | None = None,
+ module_pins: dict[str, str] | None = None,
+ module_values: dict[tuple[Key, tuple[str, ...]], float] | None = None,
+ module_deps: dict[str, list[str]] | None = None,
+) -> Plan:
+ """Decide every output's value and record; never raise on a move, collect
+ every problem instead. Pure: no engine, no files. A "fix_modules" target
+ reads its evidence (loaded and pinned by the caller), the modules' pins and
+ their values on the new engine from the keyword arguments."""
+ evidence = evidence or {}
+ module_pins = module_pins or {}
+ module_values = module_values or {}
+ plan = Plan()
+ engine = actions["engine"]
+ date = actions["date"]
+ board = base.reference.values()
+ records = {key_of(r): r for r in release["exclusions"]}
+ excluded = set(records)
+ base_excluded = {key_of(r) for r in base.exclusions["exclusions"]}
+ if not ruled <= excluded or excluded - ruled != base_excluded:
+ plan.problems.append("the release record is not 20261006's plus the ruled")
+ approved = _entries(actions, "approved")
+ regenerated = _entries(actions, "regenerated_exclusions")
+ added = _entries(actions, "new_exclusions")
+ rechecked = _entries(actions, "excluded_rechecked")
+ kept = _entries(actions, "kept_exclusions_from_release")
+ if set(computed) != set(board):
+ missing = sorted(set(board) - set(computed))[:5]
+ extra = sorted(set(computed) - set(board))[:5]
+ plan.problems.append(
+ f"computed outputs differ: missing {missing} extra {extra}"
+ )
+ for name, entries in (
+ ("approved", approved),
+ ("regenerated_exclusions", regenerated),
+ ("new_exclusions", added),
+ ("excluded_rechecked", rechecked),
+ ("kept_exclusions_from_release", kept),
+ ):
+ for k in entries:
+ if k not in board:
+ plan.problems.append(f"{name} names {k}, which is not a reference")
+ if actions["previous_engine"] != PREVIOUS_ENGINE:
+ plan.problems.append(
+ f"previous_engine is {actions['previous_engine']}, not {PREVIOUS_ENGINE}"
+ )
+ regenerated_ruled = {k for k in regenerated if k in ruled}
+ if set(kept) | regenerated_ruled != ruled:
+ missing = sorted(ruled - set(kept) - regenerated_ruled)
+ plan.problems.append(
+ "kept_exclusions_from_release and the regenerated ruled outputs must be "
+ f"the spec's ruled outputs: missing {missing}, extra "
+ f"{sorted(set(kept) - ruled)}"
+ )
+ plan.kept = sorted(k for k in kept if k in ruled)
+
+ def change(k: Key, value: float, cause: str, basis: str) -> dict:
+ return {
+ "scenario_id": k[0],
+ "variable": k[1],
+ "frozen": float(base.frozen[k]),
+ "previous": board[k],
+ "regenerated": value,
+ "cause": cause,
+ "basis": basis,
+ }
+
+ within_basis = (
+ f"Moves by $1 or less on {engine}; no score changes at the exact-match "
+ "tolerance."
+ )
+ for row in base.reference.rows:
+ k = row.key
+ if k not in computed:
+ continue
+ value, old = float(computed[k]), board[k]
+ if not math.isfinite(value):
+ plan.problems.append(f"{k}: the engine gives a non-finite value {value!r}")
+ continue
+ moves = beyond(k[1], old, value)
+ if k in excluded:
+ for name, entries in (("approved", approved), ("new_exclusions", added)):
+ if k in entries:
+ plan.problems.append(f"{name} names {k}, which is excluded")
+ record = records[k]
+ if k in regenerated:
+ entry = regenerated[k]
+ alternative = float(record["alternative_value"])
+ tolerance = float(entry.get("tolerance", -1))
+ if not math.isfinite(tolerance):
+ tolerance = -1.0
+ if record["reason_code"] != ENGINE_DEFECT:
+ plan.problems.append(
+ f"regenerated {k} is {record['reason_code']}, not an engine "
+ "defect"
+ )
+ continue
+ if abs(float(entry["alternative_value"]) - alternative) > EPS:
+ plan.problems.append(
+ f"regenerated {k}: alternative_value "
+ f"{entry['alternative_value']} is not the record's "
+ f"{alternative}"
+ )
+ continue
+ if not 0 <= tolerance <= MAX_REGENERATION_TOL:
+ plan.problems.append(
+ f"regenerated {k}: tolerance {tolerance} outside [0, "
+ f"{MAX_REGENERATION_TOL}]"
+ )
+ continue
+ target, problems = regeneration_target(
+ k,
+ entry,
+ record,
+ value,
+ tolerance,
+ engine=engine,
+ evidence=evidence,
+ module_pins=module_pins,
+ module_values=module_values,
+ module_deps=module_deps,
+ )
+ if problems:
+ plan.problems.extend(problems)
+ continue
+ # A flag must equal its target; an amount must land within the
+ # tolerance, which is at most the exact-match $1.
+ aim = target["value"]
+ if abs(value - aim) > tolerance or beyond(k[1], aim, value):
+ plan.problems.append(
+ f"regenerated {k} misses its audited target: engine "
+ f"{value!r}, target {aim!r} +/- {tolerance} "
+ f"({target['kind']})"
+ )
+ continue
+ if not str(entry.get("basis", "")).strip():
+ plan.problems.append(f"regenerated {k} has no basis")
+ continue
+ upstream = str(entry.get("upstream", "")).strip()
+ if upstream.lower() in UNNAMED_UPSTREAM:
+ plan.problems.append(
+ f"regenerated {k} names no upstream fix ({upstream!r})"
+ )
+ continue
+ if abs(value - old) > EPS:
+ plan.new_values[k] = value
+ plan.changed.append(
+ change(k, value, "regenerated_upstream_fix", entry["basis"])
+ )
+ plan.regenerated.append(
+ {
+ "scenario_id": k[0],
+ "variable": k[1],
+ "root_cause": record["root_cause"],
+ "decided_on": record["decided_on"],
+ "kept_value": old,
+ f"value_on_{engine_number(engine).replace('.', '_')}": value,
+ "audited_alternative_value": alternative,
+ "target": target,
+ "regenerated": value,
+ "tolerance": tolerance,
+ "upstream": entry["upstream"],
+ "basis": entry["basis"],
+ "record": copy.deepcopy(record),
+ }
+ )
+ continue
+ # Stays excluded: keeps the value it was decided on (rule 5).
+ if moves:
+ if k not in rechecked:
+ plan.problems.append(
+ f"excluded output moved without a recheck: {k}: "
+ f"{old!r} -> {value!r}"
+ )
+ elif not str(rechecked[k].get("reason", "")).strip():
+ plan.problems.append(f"rechecked {k} has no reason")
+ else:
+ plan.rechecked.append(
+ {
+ "scenario_id": k[0],
+ "variable": k[1],
+ "kept_value": old,
+ f"value_on_{engine_number(engine).replace('.', '_')}": (
+ value
+ ),
+ "reason": rechecked[k]["reason"],
+ }
+ )
+ elif k in rechecked:
+ plan.problems.append(
+ f"rechecked {k} does not move: {old!r} -> {value!r}"
+ )
+ continue
+ # Scored in the base and in the release.
+ for name, entries in (
+ ("regenerated_exclusions", regenerated),
+ ("excluded_rechecked", rechecked),
+ ("kept_exclusions_from_release", kept),
+ ):
+ if k in entries:
+ plan.problems.append(f"{name} names {k}, which is scored")
+ if k in added:
+ record = added[k]
+ problems = new_exclusion_problems(record, value, old, engine, date)
+ if problems:
+ plan.problems.extend(f"new exclusion {k}: {p}" for p in problems)
+ continue
+ plan.new_values[k] = value
+ plan.new_records.append(copy.deepcopy(record))
+ plan.changed.append(
+ change(
+ k,
+ value,
+ f"excluded_{record['reason_code']}",
+ f"Newly excluded from scoring on {record['decided_on']} "
+ "(reference_exclusions.json): "
+ + str(
+ record.get("unlisted_input")
+ or record.get("root_cause")
+ or record["alternative_reading"]
+ ),
+ )
+ )
+ continue
+ if k in approved:
+ entry = approved[k]
+ if not moves:
+ plan.problems.append(
+ f"approved {k} does not move: {old!r} -> {value!r}"
+ )
+ continue
+ if not math.isfinite(float(entry["value"])) or (
+ abs(value - float(entry["value"])) > APPROVED_TOL
+ ):
+ plan.problems.append(
+ f"approved {k}: engine {value!r} != approved {entry['value']!r}"
+ )
+ continue
+ if (
+ not str(entry.get("cause", "")).strip()
+ or not str(entry.get("basis", "")).strip()
+ ):
+ plan.problems.append(f"approved {k} needs a cause and a basis")
+ continue
+ plan.new_values[k] = value
+ plan.changed.append(change(k, value, entry["cause"], entry["basis"]))
+ continue
+ if moves:
+ plan.problems.append(f"unreviewed move {k}: {old!r} -> {value!r}")
+ continue
+ if abs(value - old) > EPS:
+ plan.new_values[k] = value
+ plan.within.append(
+ change(k, value, "engine_upgrade_within_1", within_basis)
+ )
+ plan.new_records.sort(key=key_of)
+ return plan
+
+
+def new_exclusion_problems(
+ record: dict, value: float, old: float, engine: str, date: str
+) -> list[str]:
+ from policybench.reference_exclusions import (
+ REASON_CODES,
+ REQUIRED_FIELDS_BY_REASON,
+ )
+
+ variable = record["variable"]
+ if record.get("reason_code") not in REASON_CODES:
+ return [f"unknown reason_code {record.get('reason_code')!r}"]
+ missing = [
+ name
+ for name in REQUIRED_FIELDS_BY_REASON[record["reason_code"]]
+ if record.get(name) in (None, "")
+ ]
+ if missing:
+ return [f"record missing {missing}"]
+ problems = []
+ if not beyond(variable, old, value):
+ problems.append(f"the output does not move: {old!r} -> {value!r}")
+ if abs(float(record["frozen_value"]) - value) > FROZEN_TOL:
+ problems.append(f"frozen_value {record['frozen_value']!r} != engine {value!r}")
+ if not beyond(
+ variable, float(record["frozen_value"]), float(record["alternative_value"])
+ ):
+ problems.append("the alternative is within the exact-match tolerance")
+ if record["engine_version"] != engine:
+ problems.append(f"engine_version {record['engine_version']!r} != {engine!r}")
+ if record["decided_on"] != date:
+ problems.append(f"decided_on {record['decided_on']!r} != {date!r}")
+ return problems
+
+
+# --- Records -------------------------------------------------------------------
+
+
+def upgrade_sentence(plan: Plan, actions: dict) -> str:
+ """The derivation's sentence naming the upgrade."""
+ engine, date = actions["engine"], actions["date"]
+ number = engine_number(engine)
+ parts = [
+ f" On {date} the references moved from {actions['previous_engine']} to"
+ f" {engine} with the same pre-freeze conventions"
+ " (reference_audit/2026-10-09-engine-upgrade), and every output was"
+ " recomputed there."
+ ]
+ n = len(plan.regenerated)
+ if n == 1:
+ parts.append(
+ " The new engine fixes the defect behind one excluded output, which"
+ " lands within $1 of its audited corrected value, so its record was"
+ " removed and the output is scored again (the reference sidecar's second"
+ " engine_upgrade revision lists it under regenerated_exclusions)."
+ )
+ elif n:
+ parts.append(
+ f" The new engine fixes the defects behind {words(n)} excluded outputs,"
+ " each landing within $1 of its audited corrected value, so their"
+ " records were removed and the outputs are scored again (the reference"
+ " sidecar's second engine_upgrade revision lists them under"
+ " regenerated_exclusions)."
+ )
+ n = len(plan.new_records)
+ if n == 1:
+ parts.append(
+ " One scored output the new engine moved was excluded that day; its"
+ f" record is computed on {number}."
+ )
+ elif n:
+ parts.append(
+ f" {words(n).capitalize()} scored outputs the new engine moved were"
+ f" excluded that day; their records are computed on {number}."
+ )
+ n = len(plan.rechecked)
+ if n == 1:
+ parts.append(
+ " Of the excluded outputs that stay excluded, one moved; it was"
+ " re-reviewed and keeps the value it was decided on"
+ " (excluded_outputs_rechecked)."
+ )
+ elif n:
+ parts.append(
+ f" Of the excluded outputs that stay excluded, {words(n)} moved; each"
+ " was re-reviewed and keeps the value it was decided on"
+ " (excluded_outputs_rechecked)."
+ )
+ else:
+ parts.append(
+ " No excluded output that stays excluded moved; each keeps the value it"
+ " was decided on."
+ )
+ return "".join(parts)
+
+
+def upgrade_exclusions(release: dict, plan: Plan, actions: dict) -> dict:
+ """The release record less the regenerated, the new records before the
+ trailing audit record, and the derivation naming the upgrade."""
+ doc = copy.deepcopy(release)
+ gone = {key_of(r) for r in plan.regenerated}
+ records = [r for r in doc["exclusions"] if key_of(r) not in gone]
+ tail = records[-1:] if records and key_of(records[-1]) == AUDIT_TAIL else []
+ body = records[: len(records) - len(tail)]
+ doc["exclusions"] = body + copy.deepcopy(plan.new_records) + tail
+ doc["derivation"] = doc["derivation"] + upgrade_sentence(plan, actions)
+ return doc
+
+
+def upgrade_revision(
+ plan: Plan,
+ actions: dict,
+ *,
+ bundle: dict,
+ fix_modules: list[dict],
+ provenance: dict,
+) -> dict:
+ engine = actions["engine"]
+ return {
+ "date": actions["date"],
+ "kind": "engine_upgrade",
+ "root_cause": "engine_upgrade_policyengine_us_"
+ + engine_number(engine).replace(".", "_"),
+ "outputs": "every scored output",
+ "rule": f"{RULE} {ENGINE_RULE} {UPGRADE_RULE}",
+ "engine_version": engine,
+ "previous_engine_version": actions["previous_engine"],
+ "policyengine_py": (
+ f"policyengine {bundle.get('policyengine_version')}, provenance only: its "
+ "certified US bundle is policyengine-us "
+ f"{bundle.get('bundled_model_version')}"
+ ),
+ "fix_modules": fix_modules,
+ "builder": (
+ "reference_audit/2026-10-09-engine-upgrade/scripts/"
+ "build_references_upgrade.py; households from "
+ "policybench.scenarios.Scenario.to_pe_household through "
+ "reference_audit/2026-09-28/scripts/sweep.py build_situation, as in the "
+ "2026-09-29 upgrade"
+ ),
+ "provenance": provenance,
+ "excluded_outputs_untouched": True,
+ "kept_exclusions_from_release": [
+ {"scenario_id": k[0], "variable": k[1]} for k in plan.kept
+ ],
+ "excluded_outputs_rechecked": plan.rechecked,
+ "regenerated_exclusions": plan.regenerated,
+ "new_exclusions": [
+ {
+ "scenario_id": r["scenario_id"],
+ "variable": r["variable"],
+ "reason_code": r["reason_code"],
+ "frozen_value": r["frozen_value"],
+ "alternative_value": r["alternative_value"],
+ "decided_on": r["decided_on"],
+ }
+ for r in plan.new_records
+ ],
+ "changed": plan.changed + plan.within,
+ }
+
+
+def upgrade_meta(
+ base_meta: dict, revision: dict, *, csv_sha256: str, bundle: dict, now: str
+) -> dict:
+ meta = copy.deepcopy(base_meta)
+ meta["policyengine_bundles"]["us"] = bundle
+ meta["reference_csv_sha256"] = csv_sha256
+ meta["regenerated_at_utc"] = now
+ meta["revisions"].append(revision)
+ return meta
+
+
+def verify_outputs(
+ base: Base,
+ plan: Plan,
+ csv_text: str,
+ exclusions: dict,
+ meta: dict,
+) -> None:
+ """The written records agree: the loader accepts the exclusion record, every
+ excluded output carries its frozen value, every unchanged row keeps its
+ bytes, and every earlier revision is kept as it was."""
+ import tempfile
+
+ import pandas as pd
+
+ from policybench.reference_exclusions import (
+ load_reference_exclusions,
+ verify_exclusions_against_reference,
+ )
+
+ def check(condition: bool, message: str) -> None:
+ if not condition:
+ raise Refusal(f"written records disagree: {message}")
+
+ with tempfile.TemporaryDirectory() as scratch:
+ path = Path(scratch) / "reference_exclusions.json"
+ path.write_text(dump_record(exclusions))
+ loaded = load_reference_exclusions(path)
+ verify_exclusions_against_reference(pd.read_csv(io.StringIO(csv_text)), loaded)
+ written = parse_reference_csv(csv_text)
+ check(
+ [r.key for r in written.rows] == [r.key for r in base.reference.rows],
+ "row order",
+ )
+ for old, new in zip(base.reference.rows, written.rows):
+ if old.key in plan.new_values:
+ check(new.value == plan.new_values[old.key], f"{old.key} value")
+ else:
+ check(new.raw == old.raw, f"{old.key} bytes")
+ check(meta["revisions"][:-1] == base.meta["revisions"], "earlier revisions")
+ check(
+ dump_record({"r": meta["revisions"][:-1]})
+ == dump_record({"r": base.meta["revisions"]}),
+ "earlier revision bytes",
+ )
+
+
+@dataclass
+class Written:
+ csv_text: str
+ exclusions: dict
+ meta: dict
+ traces: dict
+
+
+def write_outputs(
+ out_dir: Path,
+ base: Base,
+ release: dict,
+ plan: Plan,
+ actions: dict,
+ *,
+ bundle: dict,
+ fix_modules: list[dict],
+ provenance: dict,
+ traces: dict,
+ now: str,
+) -> Written:
+ if now[:10] != actions["date"]:
+ raise Refusal(
+ f"regenerated_at_utc {now} does not fall on the actions date "
+ f"{actions['date']}; pass --regenerated-at or redate the actions"
+ )
+ csv_text = render_reference_csv(base.reference, plan.new_values)
+ exclusions = upgrade_exclusions(release, plan, actions)
+ revision = upgrade_revision(
+ plan, actions, bundle=bundle, fix_modules=fix_modules, provenance=provenance
+ )
+ meta = upgrade_meta(
+ base.meta,
+ revision,
+ csv_sha256=sha256_bytes(csv_text.encode()),
+ bundle=bundle,
+ now=now,
+ )
+ verify_outputs(base, plan, csv_text, exclusions, meta)
+ out = Path(out_dir)
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "reference_outputs.csv").write_text(csv_text)
+ (out / "reference_exclusions.json").write_text(dump_record(exclusions))
+ (out / "reference_outputs.csv.meta.json").write_text(dump_record(meta))
+ (out / "reference_traces.json").write_text(json.dumps(traces, indent=1))
+ return Written(csv_text, exclusions, meta, traces)
+
+
+def write_computed(path: Path, base: Base, computed: dict[Key, float], states) -> None:
+ """Every output on the new engine, in sweep.py's columns."""
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", newline="") as sink:
+ writer = csv.writer(sink, lineterminator="\n")
+ writer.writerow(
+ [
+ "fix",
+ "scenario_id",
+ "state",
+ "variable",
+ "frozen",
+ "recomputed",
+ "delta",
+ "moved",
+ ]
+ )
+ for row in base.reference.rows:
+ if row.key not in computed:
+ continue
+ value = float(computed[row.key])
+ writer.writerow(
+ [
+ FIX_ENTRY,
+ row.key[0],
+ states.get(row.key[0], ""),
+ row.key[1],
+ repr(row.value),
+ repr(value),
+ repr(value - row.value),
+ beyond(row.key[1], row.value, value),
+ ]
+ )
+
+
+# --- Engine ------------------------------------------------------------------
+
+
+def load_module(name: str, path: Path):
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def fix_module_pins(base_meta: dict, fixes_dir: Path) -> list[dict]:
+ """The modules latest_final.py composes, each the bytes the 2026-09-29
+ revision pins, plus latest_final.py and the sales tax tables."""
+ first = [r for r in base_meta["revisions"] if r["kind"] == "engine_upgrade"][-1]
+ entries = []
+ for entry in first["fix_modules"]:
+ path = fixes_dir / entry["module"]
+ if not path.is_file() or sha256(path) != entry["sha256"]:
+ raise Refusal(f"{path} is not the module the 2026-09-29 upgrade pins")
+ entries.append(
+ {
+ "module": entry["module"],
+ "path": f"reference_audit/2026-09-28/fixes/{entry['module']}",
+ "sha256": entry["sha256"],
+ }
+ )
+ # Pinned to the bytes committed at the base commit, not to FIXES: imported
+ # in place, fixes_dir is FIXES and that comparison would be with itself.
+ entry_path = fixes_dir / f"{FIX_ENTRY}.py"
+ committed = git_blob(BASE_COMMIT, f"{FIXES_REL}/{FIX_ENTRY}.py")
+ if not entry_path.is_file() or sha256(entry_path) != sha256_bytes(committed):
+ raise Refusal(
+ f"{entry_path} is not {FIX_ENTRY}.py as committed at {BASE_COMMIT[:12]}"
+ )
+ entries.append(
+ {
+ "module": f"{FIX_ENTRY}.py",
+ "path": f"reference_audit/2026-09-28/fixes/{FIX_ENTRY}.py",
+ "sha256": sha256(entry_path),
+ }
+ )
+ tables = fixes_dir / SALES_TAX_TABLES
+ if not tables.is_file():
+ raise Refusal(
+ f"{tables} is missing: latest_c_irs_sales_tax_2025.py reads it from its "
+ f"own directory; copy {SALES_TAX_SOURCE.relative_to(ROOT)} beside the "
+ "fixes"
+ )
+ committed_tables = git_blob(BASE_COMMIT, SALES_TAX_REL)
+ if sha256(tables) != sha256_bytes(committed_tables) or sha256(
+ SALES_TAX_SOURCE
+ ) != sha256_bytes(committed_tables):
+ raise Refusal(f"{tables} is not {SALES_TAX_REL} as committed")
+ entries.append(
+ {
+ "module": SALES_TAX_TABLES,
+ "path": f"reference_audit/2026-09-22/fixes/{SALES_TAX_TABLES}",
+ "sha256": sha256(tables),
+ }
+ )
+ return entries
+
+
+def harness_pin(path: Path = HARNESS) -> dict:
+ """sweep.py, whose build_situation builds every household, must be the
+ bytes committed at the base commit."""
+ committed = git_blob(BASE_COMMIT, HARNESS_REL)
+ if not Path(path).is_file() or sha256(path) != sha256_bytes(committed):
+ raise Refusal(f"{path} is not {HARNESS_REL} as committed at {BASE_COMMIT[:12]}")
+ return {"path": HARNESS_REL, "sha256": sha256_bytes(committed)}
+
+
+def compute_outputs(system, scenarios, programs, build_situation) -> dict[Key, float]:
+ from policyengine_us import Simulation
+
+ from policybench.ground_truth import _extract_person_value, _pe_variable_for_output
+ from policybench.scenarios import scenario_from_dict
+ from policybench.spec import expand_programs_for_scenario
+
+ computed = {}
+ for _, srow in scenarios.iterrows():
+ scenario = scenario_from_dict(json.loads(srow["scenario_json"]))
+ situation = build_situation(scenario)
+ if scenario.id == "scenario_066":
+ head = next(iter(situation["people"].values()))
+ assert "weekly_hours_worked_before_lsr" in head, "builder hours mapping"
+ sim = Simulation(tax_benefit_system=system, situation=situation)
+ for variable in expand_programs_for_scenario(programs, scenario):
+ pe_variable = _pe_variable_for_output(variable, "us")
+ computed[(scenario.id, variable)] = float(
+ _extract_person_value(
+ sim.calculate(pe_variable, YEAR), scenario, variable
+ )
+ )
+ return computed
+
+
+def module_dependencies(name: str, read) -> list[str]:
+ """The sibling files a fix module loads, transitively, in discovery order;
+ ``read(file)`` gives a file's bytes (as committed at BASE_COMMIT). Refuses
+ a module whose closure cannot be established
+ (policybench.fix_module_closure)."""
+ from policybench.fix_module_closure import ClosureError, dependency_closure
+
+ try:
+ return dependency_closure(name, read)
+ except ClosureError as error:
+ raise Refusal(str(error)) from None
+
+
+def committed_fix(name: str) -> bytes:
+ """An audited fix file as committed at BASE_COMMIT."""
+ return git_blob(BASE_COMMIT, f"{AUDIT_FIXES_REL}/{name}")
+
+
+def audit_module_pins(names, fixes_dir: Path = AUDIT_FIXES) -> dict[str, str]:
+ """Each named audited fix module's sha256, and each sibling it loads
+ (module_dependencies): the bytes committed at BASE_COMMIT under
+ reference_audit/2026-09-22/fixes, and nothing else."""
+ names = sorted(set(names))
+ for name in list(names):
+ names += [d for d in module_dependencies(name, committed_fix) if d not in names]
+ pins = {}
+ for name in sorted(set(names)):
+ if Path(name).name != name or not name.endswith((".py", ".json")):
+ raise Refusal(f"a fix module is a file name ending in .py, not {name!r}")
+ path = Path(fixes_dir) / name
+ committed = sha256_bytes(git_blob(BASE_COMMIT, f"{AUDIT_FIXES_REL}/{name}"))
+ if not path.is_file() or sha256(path) != committed:
+ raise Refusal(
+ f"{path} is not {AUDIT_FIXES_REL}/{name} as committed at "
+ f"{BASE_COMMIT[:12]}"
+ )
+ pins[name] = committed
+ return pins
+
+
+def fix_module_parts(names, fixes_dir: Path = AUDIT_FIXES) -> tuple[list, list]:
+ """The Reform classes and household patches of the named fix modules, in
+ order (sweep.py's fix interface: a module defines reform, patch or both)."""
+ reforms, patches = [], []
+ for name in names:
+ module = load_module(
+ f"upgrade_fix_{Path(name).stem}", Path(fixes_dir) / name
+ )
+ reform = getattr(module, "reform", None)
+ patch = getattr(module, "patch", None)
+ if reform is None and patch is None:
+ raise Refusal(f"{name} defines neither reform nor patch")
+ if reform is not None:
+ reforms.append(reform)
+ if patch is not None:
+ patches.append(patch)
+ return reforms, patches
+
+
+def corrected_system(final, reforms):
+ """The pinned conventions, then each fix module's reform, as one system."""
+ from policyengine_core.reforms import Reform
+ from policyengine_us import CountryTaxBenefitSystem
+
+ parts = (final.reform, *reforms)
+
+ class corrected(Reform):
+ def apply(self):
+ for part in parts:
+ part.apply(self)
+
+ return CountryTaxBenefitSystem(reform=corrected)
+
+
+def compute_items(system, scenarios, keys, build_situation, patches=()) -> dict:
+ """The named outputs on a system, households built as compute_outputs
+ builds them and then edited by each patch in order."""
+ from policyengine_us import Simulation
+
+ from policybench.ground_truth import _extract_person_value, _pe_variable_for_output
+ from policybench.scenarios import scenario_from_dict
+
+ by_id = scenarios.set_index("scenario_id")
+ out = {}
+ for scenario_id in sorted({k[0] for k in keys}):
+ scenario = scenario_from_dict(
+ json.loads(by_id.loc[scenario_id, "scenario_json"])
+ )
+ situation = build_situation(scenario)
+ for patch in patches:
+ situation = patch(copy.deepcopy(situation), scenario)
+ sim = Simulation(tax_benefit_system=system, situation=situation)
+ for k in sorted(k for k in keys if k[0] == scenario_id):
+ pe_variable = _pe_variable_for_output(k[1], "us")
+ out[k] = float(
+ _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, k[1])
+ )
+ return out
+
+
+def module_values_on(final, scenarios, requests, build_situation) -> dict:
+ """Each (output, modules) request's value with the conventions and the
+ modules applied, one system per module list."""
+ values = {}
+ by_modules: dict[tuple[str, ...], list[Key]] = {}
+ for k, modules in requests:
+ by_modules.setdefault(tuple(modules), []).append(k)
+ for modules, keys in sorted(by_modules.items()):
+ reforms, patches = fix_module_parts(modules)
+ system = corrected_system(final, reforms)
+ for k, value in compute_items(
+ system, scenarios, keys, build_situation, patches
+ ).items():
+ values[(k, modules)] = value
+ return values
+
+
+def load_evidence(ref: dict) -> dict:
+ """An evidence file a target names, at its pinned sha256."""
+ if not isinstance(ref, dict) or set(ref) != {"path", "sha256"}:
+ raise Refusal(f"a target's evidence is {{path, sha256}}, not {ref!r}")
+ path = ROOT / ref["path"]
+ if not path.is_file() or sha256(path) != ref["sha256"]:
+ raise Refusal(f"{ref['path']} is missing or not the evidence its sha256 pins")
+ doc = json.loads(path.read_text())
+ if not isinstance(doc, dict) or doc.get("kind") != EVIDENCE_KIND:
+ raise Refusal(f"{ref['path']} is not {EVIDENCE_KIND}")
+ engine_number(doc.get("engine", ""))
+ if not isinstance(doc.get("modules"), dict) or not isinstance(
+ doc.get("items"), list
+ ):
+ raise Refusal(f"{ref['path']} lacks its modules or items")
+ return doc
+
+
+def write_evidence(request_path: Path, out_path: Path, fixes_dir: Path) -> dict:
+ """Evidence mode: each requested output on the installed engine, with the
+ pinned conventions alone and with its fix modules after them."""
+ from importlib.metadata import version
+
+ import pandas as pd
+
+ request = json.loads(Path(request_path).read_text())
+ items = request.get("items") if isinstance(request, dict) else None
+ if not isinstance(items, list) or not items:
+ raise Refusal("an evidence request is {\"items\": [...]} with one or more")
+ base = load_base()
+ board = base.reference.values()
+ requests = []
+ for item in items:
+ k, modules = key_of(item), tuple(item.get("modules") or ())
+ if k not in board or not modules:
+ raise Refusal(f"evidence request {k}: not a reference, or no modules")
+ requests.append((k, modules))
+ if len(set(requests)) != len(requests):
+ raise Refusal("an evidence request names an item twice")
+ pins = audit_module_pins([m for _, modules in requests for m in modules])
+ fix_module_pins(base.meta, fixes_dir)
+ harness_pinned = harness_pin()
+ harness = load_module("upgrade_sweep_harness", HARNESS)
+ final = load_module(f"upgrade_{FIX_ENTRY}", fixes_dir / f"{FIX_ENTRY}.py")
+
+ from policyengine_us import CountryTaxBenefitSystem
+
+ scenarios = pd.read_csv(io.StringIO(base.scenarios_csv))
+ plain = compute_items(
+ CountryTaxBenefitSystem(reform=final.reform),
+ scenarios,
+ [k for k, _ in requests],
+ harness.build_situation,
+ )
+ corrected = module_values_on(final, scenarios, requests, harness.build_situation)
+ doc = {
+ "kind": EVIDENCE_KIND,
+ "engine": f"policyengine-us {version('policyengine-us')}",
+ "policyengine_core": version("policyengine-core"),
+ "computed_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+ "builder_sha256": sha256(Path(__file__)),
+ "harness_sha256": harness_pinned["sha256"],
+ "conventions": f"{FIXES_REL}/{FIX_ENTRY}.py",
+ "base_commit": BASE_COMMIT,
+ "modules": pins,
+ "items": [
+ {
+ "scenario_id": k[0],
+ "variable": k[1],
+ "modules": list(modules),
+ "engine_value": plain[k],
+ "corrected_value": corrected[(k, modules)],
+ }
+ for k, modules in requests
+ ],
+ }
+ out = Path(out_path)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ out.write_text(dump_record(doc))
+ return doc
+
+
+def trace_outputs(system, scenarios, items, build_situation) -> dict:
+ import types
+
+ sys.modules.setdefault("litellm", types.ModuleType("litellm"))
+ from policyengine_us import Simulation
+
+ from policybench.case_reference_explanations import _find_target_tree, _render_trace
+ from policybench.ground_truth import _pe_variable_for_output
+ from policybench.scenarios import scenario_from_dict
+
+ by_id = scenarios.set_index("scenario_id")
+ traces = {}
+ for item in items:
+ scenario = scenario_from_dict(
+ json.loads(by_id.loc[item["scenario_id"], "scenario_json"])
+ )
+ sim = Simulation(tax_benefit_system=system, situation=build_situation(scenario))
+ sim.trace = True
+ pe_variable = _pe_variable_for_output(item["variable"], "us")
+ sim.calculate(pe_variable, YEAR)
+ tree = _find_target_tree(sim.tracer.trees, pe_variable)
+ traces[f"{item['scenario_id']}|{item['variable']}"] = {
+ "pe_variable": pe_variable,
+ "trace": "\n".join(_render_trace(tree)) if tree else "",
+ }
+ return traces
+
+
+def main(argv: list[str] | None = None) -> None:
+ from importlib.metadata import version
+
+ import pandas as pd
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--actions")
+ parser.add_argument("--out-dir")
+ parser.add_argument("--fixes-dir", default=str(FIXES))
+ parser.add_argument("--computed-csv")
+ parser.add_argument("--regenerated-at")
+ parser.add_argument("--allow-draft", action="store_true")
+ parser.add_argument("--evidence-request")
+ parser.add_argument("--evidence-out")
+ args = parser.parse_args(argv)
+
+ if args.evidence_request or args.evidence_out:
+ if not (args.evidence_request and args.evidence_out) or args.actions:
+ parser.error("evidence mode takes --evidence-request and --evidence-out")
+ doc = write_evidence(
+ Path(args.evidence_request),
+ Path(args.evidence_out),
+ Path(args.fixes_dir).resolve(),
+ )
+ print(f"{doc['engine']}: {len(doc['items'])} items")
+ for item in doc["items"]:
+ print(
+ f" {item['scenario_id']} {item['variable']:46s} "
+ f"{item['engine_value']:>11.2f} -> {item['corrected_value']:>11.2f} "
+ f"{'+'.join(item['modules'])}"
+ )
+ return
+ if not (args.actions and args.out_dir):
+ parser.error("a build takes --actions and --out-dir")
+
+ actions_path = Path(args.actions)
+ actions = json.loads(actions_path.read_text())
+ validate_actions(actions, allow_draft=args.allow_draft)
+ engine = engine_number(actions["engine"])
+ installed = version("policyengine-us")
+ if installed != engine:
+ raise Refusal(f"need policyengine-us {engine}, have {installed}")
+
+ base = load_base()
+ release, ruled, spec_sha = release_record(base.exclusions)
+ if actions.get("release_spec_sha256") != spec_sha:
+ drafted = actions.get("release_spec_sha256")
+ raise Refusal(
+ f"the actions were drafted against spec {drafted}, not "
+ f"docs/haiku55/spec.json {spec_sha}"
+ )
+ fixes_dir = Path(args.fixes_dir).resolve()
+ fix_modules = fix_module_pins(base.meta, fixes_dir)
+ harness_pinned = harness_pin()
+ harness = load_module("upgrade_sweep_harness", HARNESS)
+ final = load_module(f"upgrade_{FIX_ENTRY}", fixes_dir / f"{FIX_ENTRY}.py")
+
+ from policyengine_us import CountryTaxBenefitSystem
+
+ from policybench.policyengine_runtime import policyengine_release_bundle
+
+ system = CountryTaxBenefitSystem(reform=final.reform)
+ scenarios = pd.read_csv(io.StringIO(base.scenarios_csv))
+ computed = compute_outputs(
+ system, scenarios, base.meta["programs"], harness.build_situation
+ )
+ states = dict(zip(scenarios["scenario_id"], scenarios["state"]))
+ out = Path(args.out_dir)
+ write_computed(
+ Path(args.computed_csv or out / "computed.csv"), base, computed, states
+ )
+ targets = [
+ entry
+ for entry in actions.get("regenerated_exclusions", [])
+ if isinstance(entry.get("target"), dict)
+ and entry["target"].get("kind") == TARGET_FIX_MODULES
+ ]
+ evidence = {
+ entry["target"]["evidence"]["path"]: load_evidence(entry["target"]["evidence"])
+ for entry in targets
+ if isinstance(entry["target"].get("evidence"), dict)
+ }
+ named = sorted({m for entry in targets for m in target_modules(entry)})
+ module_pins = audit_module_pins(named)
+ module_deps = {m: module_dependencies(m, committed_fix) for m in named}
+ module_values = module_values_on(
+ final,
+ scenarios,
+ [(key_of(entry), target_modules(entry)) for entry in targets],
+ harness.build_situation,
+ )
+ plan = plan_upgrade(
+ base,
+ release,
+ ruled,
+ actions,
+ computed,
+ evidence=evidence,
+ module_pins=module_pins,
+ module_values=module_values,
+ module_deps=module_deps,
+ )
+ if plan.problems:
+ raise Refusal("refusing to write references:\n " + "\n ".join(plan.problems))
+
+ traces = trace_outputs(
+ system, scenarios, plan.changed + plan.within, harness.build_situation
+ )
+ now = (
+ args.regenerated_at or datetime.datetime.now(datetime.timezone.utc).isoformat()
+ )
+ provenance = {
+ "builder_sha256": sha256(Path(__file__)),
+ "harness": harness_pinned["path"],
+ "harness_sha256": harness_pinned["sha256"],
+ "actions_sha256": sha256(actions_path),
+ "release_spec_sha256": spec_sha,
+ "base_commit": BASE_COMMIT,
+ "policyengine_core": version("policyengine-core"),
+ }
+ written = write_outputs(
+ out,
+ base,
+ release,
+ plan,
+ actions,
+ bundle=policyengine_release_bundle("us"),
+ fix_modules=fix_modules,
+ provenance=provenance,
+ traces=traces,
+ now=now,
+ )
+ print(f"sha256 {written.meta['reference_csv_sha256']}")
+ print(
+ f"{len(plan.changed)} reviewed changes, {len(plan.within)} within $1, "
+ f"{len(plan.regenerated)} regenerated exclusions, {len(plan.new_records)} new "
+ f"exclusions, {len(plan.rechecked)} excluded outputs rechecked, "
+ f"{len(written.exclusions['exclusions'])} exclusion records"
+ )
+ for c in plan.changed + plan.within:
+ print(
+ f" {c['scenario_id']} {c['variable']:46s} {c['previous']:>11.2f} -> "
+ f"{c['regenerated']:>11.2f} {c['cause']}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/reference_audit/2026-10-09-engine-upgrade/scripts/check_against_evidence.py b/reference_audit/2026-10-09-engine-upgrade/scripts/check_against_evidence.py
new file mode 100644
index 00000000..00f004e3
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/scripts/check_against_evidence.py
@@ -0,0 +1,111 @@
+"""Check an engine against the fix modules' pre-fix evidence, before it ships.
+
+For each evidence item (reference_audit/2026-10-09-engine-upgrade/evidence/),
+compute the output on the importable policyengine-us, with the pinned
+conventions alone and with the item's fix modules after them, and report:
+
+ lands the engine is within $1 of the evidence's corrected value (a flag
+ equals it), which a regeneration needs;
+ inert the modules move it by no more than $1 there: nothing is left to fix.
+
+This is the builder's regeneration gate (build_references_upgrade.py,
+regeneration_target) run early, so a policyengine-us pull request can be checked
+before it merges. PYTHONPATH may put a policyengine-us checkout ahead of the
+installed one; the engine is then labelled by --label, since the installed
+package's version would misname it. Nothing is written; the exit code is 1 when
+any checked item fails.
+
+ PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 \\
+ PYTHONPATH=: \\
+ /bin/python .../check_against_evidence.py \\
+ --evidence reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json \\
+ --label "PR #10032 head abc1234" [--module r02_ira_219g_v2.py ...]
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+import build_references_upgrade as build # noqa: E402
+
+TOL = 1.0
+
+
+def main(argv: list[str] | None = None) -> int:
+ import pandas as pd
+ import policyengine_us
+ from policyengine_us import CountryTaxBenefitSystem
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--evidence", required=True)
+ parser.add_argument("--label", required=True)
+ parser.add_argument(
+ "--module",
+ action="append",
+ default=[],
+ help="check only items whose modules include this one (repeatable)",
+ )
+ parser.add_argument("--fixes-dir", default=str(build.FIXES))
+ args = parser.parse_args(argv)
+
+ path = build.ROOT / args.evidence
+ doc = build.load_evidence({"path": args.evidence, "sha256": build.sha256(path)})
+ items = [
+ item
+ for item in doc["items"]
+ if not args.module or set(args.module) & set(item["modules"])
+ ]
+ if not items:
+ raise build.Refusal("no evidence item uses the named modules")
+ pins = build.audit_module_pins([m for item in items for m in item["modules"]])
+ stale = sorted(m for m, pin in pins.items() if doc["modules"].get(m) != pin)
+ if stale:
+ raise build.Refusal(f"the evidence ran other bytes of {stale}")
+ base = build.load_base()
+ fixes_dir = Path(args.fixes_dir).resolve()
+ build.fix_module_pins(base.meta, fixes_dir)
+ build.harness_pin()
+ harness = build.load_module("upgrade_sweep_harness", build.HARNESS)
+ final = build.load_module(f"upgrade_{build.FIX_ENTRY}", fixes_dir / "latest_final.py")
+ scenarios = pd.read_csv(io.StringIO(base.scenarios_csv))
+ requests = [(build.key_of(item), tuple(item["modules"])) for item in items]
+ plain = build.compute_items(
+ CountryTaxBenefitSystem(reform=final.reform),
+ scenarios,
+ [k for k, _ in requests],
+ harness.build_situation,
+ )
+ corrected = build.module_values_on(
+ final, scenarios, requests, harness.build_situation
+ )
+ print(
+ f"{args.label} (policyengine_us from {Path(policyengine_us.__file__).parent}) "
+ f"against {doc['engine']} evidence"
+ )
+ failed = 0
+ for item, (k, modules) in zip(items, requests):
+ value, after = plain[k], corrected[(k, modules)]
+ target = float(item["corrected_value"])
+ lands = abs(value - target) <= TOL and not build.beyond(k[1], target, value)
+ inert = abs(after - value) <= TOL and not build.beyond(k[1], value, after)
+ failed += not (lands and inert)
+ print(
+ f" {'PASS' if lands and inert else 'FAIL'} {k[0]} {k[1]:44s} "
+ f"engine {value:>11.2f} target {target:>11.2f} "
+ f"{'lands' if lands else 'MISSES by ' + format(value - target, '+.2f')}; "
+ f"{'inert' if inert else 'modules still move it to ' + format(after, '.2f')}"
+ f" [{'+'.join(modules)}]"
+ )
+ print(f"{len(items) - failed} of {len(items)} pass")
+ return 1 if failed else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/reference_audit/2026-10-09-engine-upgrade/scripts/narratives_upgrade.py b/reference_audit/2026-10-09-engine-upgrade/scripts/narratives_upgrade.py
new file mode 100644
index 00000000..24c11a77
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/scripts/narratives_upgrade.py
@@ -0,0 +1,429 @@
+"""Rewrite the reference explanations of the outputs the second engine upgrade
+changed; every other row of the explanations CSV keeps its bytes.
+
+Adapted from reference_audit/2026-09-28/scripts/narratives_latest.py. The writer
+is every reference narrative's (policybench.case_reference_explanations:
+REFERENCE_MODEL claude-haiku-4-5, TEMPERATURE, MAX_TOKENS, _prompt), given the
+build's engine trace (reference_traces.json from build_references_upgrade.py) and
+the change's recorded basis as grounding. A new exclusion is grounded on its
+record's alternative reading, as in 2026-09-28. The narrative must state the new
+reference value (an amount; a flag states none); a draft that omits it is retried
+twice, then refused. --hand-corrected takes a JSON object
+{"scenario_id|variable": text} that replaces the writer for a listed output; its
+text must also state the value.
+
+--reuse-from --reuse-explanations keeps
+an earlier build's narrative for an output whose writer inputs are the same in
+both builds: the value, the change's cause and grounding, the PolicyEngine
+variable and the trace, byte for byte. On another engine the narrative must
+also not name the earlier engine's version. Judges' prompts render the narrative, so a reused one
+keeps a case's verdict through a rebuild on a newer engine. The reused rows are
+listed beside --out (.reused.json) with both builds' sidecar sha256.
+
+The rows rewritten are exactly the sidecar's last revision's changed list (a
+reference that changed by more than EPS); each gets the new reference_value,
+trace_lines and explanation, and an empty error. Judge prompts render a case's
+reference value and explanation, so these are the cases whose verdicts
+prepare_audit drops and the judge re-runs.
+
+The writer is Anthropic's, through litellm: the run needs ANTHROPIC_API_KEY and
+never an OpenAI key (the script removes OPENAI_API_KEY from its environment):
+
+ ANTHROPIC_API_KEY=$(agent-secret get ANTHROPIC_API_KEY maxghenis) \\
+ PYTHONPATH=/bin/python narratives_upgrade.py \\
+ --references \\
+ --explanations annotations//us_case_reference_explanations.csv \\
+ --out
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import io
+import json
+import math
+import os
+import re
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+from build_references_upgrade import ( # noqa: E402
+ BASE_COMMIT,
+ BASE_SHA256,
+ RUN,
+ YEAR,
+ Refusal,
+ format_row,
+ git_blob,
+ physical_lines,
+ sha256_bytes,
+)
+
+FIELDS = [
+ "country",
+ "scenario_id",
+ "variable",
+ "reference_value",
+ "trace_lines",
+ "explanation",
+ "error",
+]
+RETRIES = 2
+
+
+def money(value: float) -> str:
+ return f"{value:,.2f}".removesuffix(".00")
+
+
+# A dollar amount as a narrative writes it: "$1,916.61", "$0", "-$12.50".
+DOLLARS = re.compile(r"(-)?\$\s?(\d[\d,]*(?:\.\d+)?)")
+
+
+def required(item: dict) -> list[float]:
+ """What the narrative must state: the new amount; nothing for a flag."""
+ if item["variable"].endswith("_eligible"):
+ return []
+ return [float(item["regenerated"])]
+
+
+def states_amount(text: str, value: float) -> bool:
+ """Whether ``text`` states ``value`` as a dollar amount, to the cent. A
+ bare number does not count: for a $0 reference, "2026" states nothing."""
+ for sign, number in DOLLARS.findall(text):
+ amount = float(number.replace(",", "")) * (-1 if sign else 1)
+ if abs(amount - value) <= 0.005 + 1e-9:
+ return True
+ return False
+
+
+def missing_amounts(item: dict, text: str) -> list[str]:
+ return [money(v) for v in required(item) if not states_amount(text, v)]
+
+
+def engine_note(engine: str) -> str:
+ return (
+ f" PolicyEngine here is {engine} with the benchmark's conventions for law "
+ "published before July 3, 2026."
+ )
+
+
+def grounding_for(item: dict, records: dict) -> str:
+ """The change's recorded basis; a new exclusion's record reading instead."""
+ record = records.get((item["scenario_id"], item["variable"]))
+ if record is not None and item["cause"] == f"excluded_{record['reason_code']}":
+ if record["reason_code"] == "reference_depends_on_unlisted_input":
+ return (
+ "PolicyBench does not score this output, because the reference "
+ "depends on an input the prompt does not state. "
+ + record["alternative_reading"]
+ )
+ return (
+ "PolicyBench does not score this output. " + record["alternative_reading"]
+ )
+ return item["basis"]
+
+
+def parse_rows(text: str) -> tuple[str, list[tuple[list[str], str]]]:
+ """The header line and each record with its raw text."""
+ lines = physical_lines(text)
+ reader = csv.reader(iter(lines))
+ records, used = [], 0
+ for fields in reader:
+ records.append((fields, "".join(lines[used : reader.line_num])))
+ used = reader.line_num
+ (header, header_raw), body = records[0], records[1:]
+ if header != FIELDS:
+ raise Refusal(f"unexpected explanations header {header}")
+ return header_raw, body
+
+
+def rewrite_explanations(text: str, rewrites: dict) -> str:
+ """``text`` with each listed (scenario_id, variable) row replaced by
+ (reference_value, trace_lines, explanation); every other row byte for byte."""
+ header_raw, body = parse_rows(text)
+ keys = [(fields[1], fields[2]) for fields, _ in body]
+ for key in rewrites:
+ if keys.count(key) != 1:
+ raise Refusal(f"{key} is in the explanations {keys.count(key)} times")
+ parts = [header_raw]
+ for fields, raw in body:
+ key = (fields[1], fields[2])
+ if key not in rewrites:
+ parts.append(raw)
+ continue
+ value, n_lines, explanation = rewrites[key]
+ row = format_row(
+ [
+ fields[0],
+ key[0],
+ key[1],
+ repr(float(value)),
+ str(n_lines),
+ explanation,
+ "",
+ ]
+ )
+ parts.append(row if raw.endswith("\n") else row.removesuffix("\n"))
+ out = "".join(parts)
+ # Every row the rewrite does not name keeps its bytes.
+ _, written = parse_rows(out)
+ for (old, old_raw), (new, new_raw) in zip(body, written, strict=True):
+ if (old[1], old[2]) not in rewrites and old_raw != new_raw:
+ raise Refusal(f"row {(old[1], old[2])} changed")
+ return out
+
+
+def strip_heading(text: str) -> str:
+ text = text.strip()
+ lines = text.split("\n")
+ if lines and lines[0].lstrip().startswith("#"):
+ text = "\n".join(lines[1:]).strip()
+ return text
+
+
+def write_narratives(
+ references: Path,
+ explanations_text: str,
+ scenarios,
+ *,
+ completion,
+ hand_corrected: dict | None = None,
+ reuse: dict | None = None,
+) -> tuple[str, list[tuple]]:
+ """The explanations CSV with every changed output's narrative rewritten,
+ and the rewritten rows. ``completion`` is litellm.completion (mocked in
+ tests). ``reuse`` maps an output to the earlier narrative reusable_narratives
+ found for it; the writer is not called for those."""
+ from policybench.case_reference_explanations import (
+ MAX_TOKENS,
+ REFERENCE_MODEL,
+ TEMPERATURE,
+ _prompt,
+ _scenario_summary,
+ )
+
+ if not REFERENCE_MODEL.startswith("claude-"):
+ raise Refusal(
+ f"the narrative writer must be Anthropic's, not {REFERENCE_MODEL}"
+ )
+ meta = json.loads((references / "reference_outputs.csv.meta.json").read_text())
+ upgrade = meta["revisions"][-1]
+ if (
+ upgrade["kind"] != "engine_upgrade"
+ or len([r for r in meta["revisions"] if r["kind"] == "engine_upgrade"]) < 2
+ ):
+ raise Refusal("the sidecar's last revision is not the second engine upgrade")
+ traces = json.loads((references / "reference_traces.json").read_text())
+ exclusions = json.loads((references / "reference_exclusions.json").read_text())
+ records = {(e["scenario_id"], e["variable"]): e for e in exclusions["exclusions"]}
+ note = engine_note(upgrade["engine_version"])
+ hand_corrected = hand_corrected or {}
+ reuse = reuse or {}
+ by_id = scenarios.set_index("scenario_id", drop=False)
+
+ def write(item: dict, extra: str = "") -> str:
+ key = (item["scenario_id"], item["variable"])
+ traced = traces[f"{key[0]}|{key[1]}"]
+ prompt = _prompt(
+ "us",
+ _scenario_summary(by_id.loc[key[0]]),
+ key[1],
+ traced["pe_variable"],
+ item["regenerated"],
+ YEAR,
+ traced["trace"],
+ grounding=grounding_for(item, records) + note + extra,
+ )
+ response = completion(
+ model=REFERENCE_MODEL,
+ messages=[{"role": "user", "content": prompt}],
+ temperature=TEMPERATURE,
+ max_tokens=MAX_TOKENS,
+ )
+ return strip_heading(response.choices[0].message.content)
+
+ rewrites, results = {}, []
+ for item in upgrade["changed"]:
+ key = (item["scenario_id"], item["variable"])
+ n_lines = len(traces[f"{key[0]}|{key[1]}"]["trace"].splitlines())
+ listed = f"{key[0]}|{key[1]}"
+ if listed in hand_corrected:
+ text = hand_corrected[listed]
+ elif key in reuse:
+ text = reuse[key]
+ else:
+ text = write(item)
+ for _ in range(RETRIES):
+ missing = missing_amounts(item, text)
+ if not missing:
+ break
+ text = write(
+ item,
+ " The narrative must state the value "
+ + ", ".join(f"${m}" for m in missing)
+ + ".",
+ )
+ missing = missing_amounts(item, text)
+ if missing:
+ raise Refusal(f"{key[0]} {key[1]}: the narrative omits {missing}")
+ rewrites[key] = (item["regenerated"], n_lines, text)
+ results.append((key[0], key[1], item["regenerated"], n_lines, text))
+ unused = set(hand_corrected) - {f"{k[0]}|{k[1]}" for k in rewrites}
+ if unused:
+ raise Refusal(f"hand-corrected outputs the upgrade did not change: {unused}")
+ return rewrite_explanations(explanations_text, rewrites), results
+
+
+def writer_inputs(references: Path) -> dict:
+ """Each changed output's narrative-writer inputs in a build, by output:
+ value, cause, grounding, PolicyEngine variable and trace."""
+ meta = json.loads((references / "reference_outputs.csv.meta.json").read_text())
+ traces = json.loads((references / "reference_traces.json").read_text())
+ exclusions = json.loads((references / "reference_exclusions.json").read_text())
+ records = {(e["scenario_id"], e["variable"]): e for e in exclusions["exclusions"]}
+ inputs = {}
+ for item in meta["revisions"][-1]["changed"]:
+ key = (item["scenario_id"], item["variable"])
+ traced = traces[f"{key[0]}|{key[1]}"]
+ inputs[key] = (
+ repr(float(item["regenerated"])),
+ item["cause"],
+ grounding_for(item, records),
+ traced["pe_variable"],
+ traced["trace"],
+ )
+ return inputs
+
+
+def reusable_narratives(
+ references: Path, earlier: Path, earlier_explanations_text: str
+) -> dict:
+ """The earlier build's narratives for outputs whose writer inputs both
+ builds share and whose narrative does not name the earlier engine."""
+ def engine(path: Path) -> str:
+ meta = json.loads((path / "reference_outputs.csv.meta.json").read_text())
+ return meta["revisions"][-1]["engine_version"]
+
+ now, then = engine(references), engine(earlier)
+ number = then.removeprefix("policyengine-us ")
+ current = writer_inputs(references)
+ # The groundings name their build's engine; that name may differ, nothing else.
+ before = {
+ key: (*inputs[:2], inputs[2].replace(then, now), *inputs[3:])
+ for key, inputs in writer_inputs(earlier).items()
+ }
+ _, body = parse_rows(earlier_explanations_text)
+ rows: dict = {}
+ for fields, _ in body:
+ rows.setdefault((fields[1], fields[2]), []).append(fields)
+ # The CSV must be the earlier build's narratives: each output that build
+ # changed appears once, for the US, with no error, at that build's value.
+ # Each value must parse to a finite number: abs(nan - x) > 1e-6 is false,
+ # so a "nan" would pass a bare comparison.
+ wrong = sorted(
+ key
+ for key, inputs in writer_inputs(earlier).items()
+ if len(rows.get(key, [])) != 1
+ or rows[key][0][0] != "us"
+ or rows[key][0][6] != ""
+ or (stated := finite(rows[key][0][3])) is None
+ or (value := finite(inputs[0])) is None
+ or abs(stated - value) > 1e-6
+ )
+ if wrong:
+ raise Refusal(
+ "--reuse-explanations is not --reuse-from's narratives: "
+ f"{[f'{k[0]}|{k[1]}' for k in wrong[:5]]}"
+ )
+ narratives = {key: found[0][5] for key, found in rows.items()}
+ # A narrative naming the earlier engine is stale on another engine, and
+ # still true on the same one.
+ return {
+ key: narratives[key]
+ for key, inputs in current.items()
+ if before.get(key) == inputs and (then == now or number not in narratives[key])
+ }
+
+
+def finite(text) -> float | None:
+ """A value's text as a finite float, or None (unparsable, nan or inf)."""
+ try:
+ value = float(text)
+ except (TypeError, ValueError):
+ return None
+ return value if math.isfinite(value) else None
+
+
+def main(argv: list[str] | None = None) -> None:
+ import pandas as pd
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--references", required=True)
+ parser.add_argument("--explanations", required=True)
+ parser.add_argument("--out", required=True)
+ parser.add_argument("--hand-corrected")
+ parser.add_argument("--reuse-from")
+ parser.add_argument("--reuse-explanations")
+ args = parser.parse_args(argv)
+ if bool(args.reuse_from) != bool(args.reuse_explanations):
+ parser.error("--reuse-from and --reuse-explanations go together")
+ os.environ.pop("OPENAI_API_KEY", None)
+ if not os.environ.get("ANTHROPIC_API_KEY"):
+ raise Refusal(
+ "set ANTHROPIC_API_KEY (agent-secret get ANTHROPIC_API_KEY maxghenis)"
+ )
+ import litellm
+
+ raw = git_blob(BASE_COMMIT, f"{RUN}/scenarios.csv")
+ if sha256_bytes(raw) != BASE_SHA256["scenarios.csv"]:
+ raise Refusal("scenarios.csv at the base commit does not match its pin")
+ scenarios = pd.read_csv(io.StringIO(raw.decode()))
+ hand = (
+ json.loads(Path(args.hand_corrected).read_text()) if args.hand_corrected else {}
+ )
+ reuse = {}
+ if args.reuse_from:
+ reuse = reusable_narratives(
+ Path(args.references),
+ Path(args.reuse_from),
+ Path(args.reuse_explanations).read_text(),
+ )
+ reuse = {k: v for k, v in reuse.items() if f"{k[0]}|{k[1]}" not in hand}
+ text, results = write_narratives(
+ Path(args.references),
+ Path(args.explanations).read_text(),
+ scenarios,
+ completion=litellm.completion,
+ hand_corrected=hand,
+ reuse=reuse,
+ )
+ Path(args.out).write_text(text)
+ if args.reuse_from:
+ sidecar = "reference_outputs.csv.meta.json"
+ Path(args.out + ".reused.json").write_text(
+ json.dumps(
+ {
+ "reused": [f"{k[0]}|{k[1]}" for k in sorted(reuse)],
+ "from_sidecar_sha256": sha256_bytes(
+ (Path(args.reuse_from) / sidecar).read_bytes()
+ ),
+ "into_sidecar_sha256": sha256_bytes(
+ (Path(args.references) / sidecar).read_bytes()
+ ),
+ },
+ indent=2,
+ )
+ + "\n"
+ )
+ print(f"reused {len(reuse)} narratives from {args.reuse_from}")
+ for scenario_id, variable, value, _, narrative in results:
+ print(f"--- {scenario_id} {variable} ({value:,.2f})\n{narrative}\n")
+ print(f"rewrote {len(results)} narratives -> {args.out}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/reference_audit/2026-10-09-engine-upgrade/scripts/probe_cells.py b/reference_audit/2026-10-09-engine-upgrade/scripts/probe_cells.py
new file mode 100644
index 00000000..8d3ad5b6
--- /dev/null
+++ b/reference_audit/2026-10-09-engine-upgrade/scripts/probe_cells.py
@@ -0,0 +1,20 @@
+"""Compute named outputs, with the pinned conventions, on the policyengine-us
+that PYTHONPATH puts first: probe_cells.py